finished e2e test suite

.dockerignore

+__pycache__

Dockerfile

+FROM jdonnal/joule:latest
+
+MAINTAINER John Donnal <donnal@usna.edu>
+
+ADD . /joule-modules
+ADD ./e2e /etc/joule
+
+CMD /usr/local/bin/jouled
+
+

e2e/bootstrap-inner.py

+#!/usr/bin/python3
+
+import sys
+import subprocess
+import os
+import signal
+
+FORCE_DUMP = False
+
+
+def main():
+    jouled = subprocess.Popen(["jouled", "--config",
+                               "/etc/joule/main.conf"],
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT,
+                              universal_newlines=True)
+    print("---------[running e2e test suite]---------")
+    sys.stdout.flush()
+    test = subprocess.run(["python3", os.path.join("/etc/joule/test.py")])
+    jouled.send_signal(signal.SIGINT)
+    if(test.returncode != 0 or FORCE_DUMP):
+        print("----dump from jouled----")
+        stdout, _ = jouled.communicate()
+        for line in stdout.rstrip().split('\n'):
+            print("> %s" % line)
+        return test.returncode
+
+    
+if __name__ == "__main__":
+    exit(main())

e2e/docker-compose.yml

+version: '2'
+services:
+  joule:
+    build: ../.
+    links:
+      - nilmdb
+    entrypoint: python3 /etc/joule/bootstrap-inner.py
+  nilmdb:
+    image: jdonnal/nilmdb
+    logging:
+      driver: none
+    stop_signal: SIGKILL

e2e/main.conf

-[Jouled]
-ModuleDirectory = example_configs/module_configs
-StreamDirectory = example_configs/stream_configs
+[NilmDB]
+url=http://nilmdb
+InsertionPeriod=1
\ No newline at end of file

e2e/module_configs/filter.conf

 [Main]
 [Main]
-exec_cmd = python3 filter.py 
+exec_cmd = python3 /joule-modules/filter.py 2
 name = Demo Filter
 name = Demo Filter
 
 
 [Source]
 [Source]

e2e/module_configs/reader.conf

 [Main]
 [Main]
-exec_cmd = python3 reader.py 
+exec_cmd = python3 /joule-modules/reader.py 0.1
 name = Demo Reader
 name = Demo Reader
 
 
 [Source]
 [Source]

e2e/runner.sh

+#!/bin/bash
+
+docker-compose up --abort-on-container-exit --build
+docker-compose rm -f

e2e/stream_configs/filtered.conf

 [Main]
 [Main]
+name = Filtered Data
 path = /demo/filtered
 path = /demo/filtered
 datatype = int32
 datatype = int32
 keep = 1w
 keep = 1w
+decimate = yes
 
 
 [Element1]
 [Element1]
-name = counter
+name = filtered counter

e2e/stream_configs/raw.conf

 [Main]
 [Main]
+name = Raw Data
 path = /demo/raw
 path = /demo/raw
 datatype = int32
 datatype = int32
 keep = 1w
 keep = 1w
+decimate = yes
 
 
 [Element1]
 [Element1]
 name = counter
 name = counter

e2e/test.py

+import time
+import numpy as np
+import re
+from joule.testing.e2eutils import joule
+from joule.testing.e2eutils import nilmtool
+import sys
+
+
+def main():
+    time.sleep(8)  # wait for jouled to boot and get data
+    check_modules()
+    check_data()
+    check_logs()
+
+
+def check_modules():
+    """
+    Test: check module status
+    Goal:
+      Demo Filter: running some nonzero memory
+      Demo Reader: running some nonzero memory
+    """
+    modules = joule.modules()
+    assert(len(modules) == 2)  # normal1,normal2,filter,broken
+    for module in modules:
+        title = module[joule.MODULES_TITLE_FIELD]
+        status = module[joule.MODULES_STATUS_FIELD]
+        if(title['name'] in ['Demo Filter', 'Demo Reader']):
+            assert status == joule.MODULES_STATUS_RUNNING, "%s not running"%title
+        else:
+            assert(0)  # unexpected module in status report
+
+
+def check_data():
+    """
+    Test: check data inserted into nilmdb
+    Goal:
+      /demo/raw is float32_1, has 1 interval with >50 samples
+      /demo/filtered is float32_2, has 1 interval with >50 samples
+      both filtered and raw have decimations
+    """
+    for path in ["/demo/raw", "/demo/filtered"]:
+        # 1.) check streams have one continuous interval
+        base_intervals = nilmtool.intervals(path)
+        decim_intervals = nilmtool.intervals(
+            path + "~decim-16")  # check level 2 decimation
+        assert len(base_intervals) == 1,\
+            "%s has %d intervals" % (path, len(base_intervals))
+        assert len(decim_intervals) == 1,\
+            "%s has %d intervals" % (path+"~decim-16", len(decim_intervals))
+        # 2.) make sure this interval has data in it
+        num_samples = nilmtool.data_count(path)
+        assert(num_samples > 50)
+        # 3.) make sure decimations have data
+        assert(nilmtool.is_decimated(path, level=16, min_size=2))
+
+    # verify stream layouts
+    assert nilmtool.layout("/demo/raw") == "int32_1"
+    assert nilmtool.layout("/demo/filtered") == "int32_1"
+
+    # verify the filter module executed correctly
+    # check the first 50 rows, the filter won't have
+    # all the source data because the process was stopped
+    expected_data = nilmtool.data_extract("/demo/raw")
+    expected_data[:, 1:] += 2
+    actual_data = nilmtool.data_extract("/demo/filtered")
+    np.testing.assert_almost_equal(
+        actual_data[:50, :], expected_data[:50, :])
+
+    
+def check_logs():
+    """
+    Test: logs should contain info and stderr from modules
+    Goal:
+      Demo Filter: says "starting" somewhere once
+      Demo Reader: says "starting" somewhere once
+    """
+    for module_name in ["Demo Filter", "Demo Reader"]:
+        logs = joule.logs(module_name)
+        num_starts = len(re.findall(joule.LOG_STARTING_STRING, logs))
+        assert(num_starts == 1)
+
+        
+if __name__ == "__main__":
+    
+    main()
+    
+    print("OK")
+

filter.py

@@ -10,7 +10,7 @@ class FilterDemo(FilterModule):
         self.help = "a paragraph: this filter does x,y,z etc..."
         self.help = "a paragraph: this filter does x,y,z etc..."
     
     
     def custom_args(self, parser):
     def custom_args(self, parser):
-        parser.add_argument("offset", type=float, default=0,
+        parser.add_argument("offset", type=int, default=0,
                             help="apply an offset")
                             help="apply an offset")
         
         
     async def run(self, parsed_args, inputs, outputs):
     async def run(self, parsed_args, inputs, outputs):

reader.py

@@ -3,6 +3,7 @@ from joule.client import ReaderModule
 import asyncio
 import asyncio
 import numpy as np
 import numpy as np
 
 
+
 class ReaderDemo(ReaderModule):
 class ReaderDemo(ReaderModule):
     "Example reader: generates incrementing values at user specified rate"
     "Example reader: generates incrementing values at user specified rate"