added support for joule along with nilmdb

app/adapters/data_builder.rb

+class DataBuilder
+
+  def self.build_raw_data(elements, resp)
+    data = elements.map { |e| { id: e.id, type: 'raw', values: [] } }
+    resp.each do |row|
+      if row.nil? # add an interval break to all the elements
+        data.each { |d| d[:values].push(nil) }
+        next
+      end
+      ts = row[0]
+      elements.each_with_index do |elem, i|
+        data[i][:values].push([ts, self.scale_value(row[1 + elem.column], elem)])
+      end
+    end
+    data
+  end
+
+  def self.build_decimated_data(elements, resp)
+    # if elements is empty we don't need to do anything
+    return [] if elements.empty?
+
+    #prepare the data structure
+    data = elements.map { |e| { id: e.id, type: 'decimated', values: Array.new(resp.length) } }
+
+    #set up constants so we compute them once
+    mean_offset = 0
+    min_offset = elements.first.db_stream.db_elements.length
+    max_offset = elements.first.db_stream.db_elements.length * 2
+
+    resp.each_with_index do |row, k|
+      if row.nil? # add an interval break to all the elements
+        data.each { |d| d[:values][k]=nil }
+        next
+      end
+      ts = row[0]
+      elements.each_with_index do |elem, i|
+        #mean = __scale_value(row[1 + elem.column + mean_offset], elem)
+        #min =  __scale_value(row[1 + elem.column + min_offset], elem)
+        #max =  __scale_value(row[1 + elem.column + max_offset], elem)
+        mean = (row[1 + elem.column + mean_offset] - elem.offset) * elem.scale_factor
+        min =  (row[1 + elem.column + min_offset] - elem.offset) * elem.scale_factor
+        max = (row[1 + elem.column + max_offset] - elem.offset) * elem.scale_factor
+        tmp_min = [min, max].min
+        max = [min, max].max
+        min = tmp_min
+        data[i][:values][k]=[ts, mean, min, max]
+      end
+    end
+    data
+  end
+
+  def self.build_interval_data(elements, resp)
+    elements.map { |e| { id: e.id, type: 'interval', values: resp } }
+  end
+
+  # for data that cannot be represented as decimations
+  # eg: events, compute intervals from the actual decimated data
+  def self.build_intervals_from_decimated_data(elements, resp)
+
+    # if elements is empty we don't need to do anything
+    return [] if elements.empty?
+    # compute intervals from resp
+    if resp.empty?
+      elements.map do |e|
+        { id: e.id,
+          type: 'interval',
+          values: [] }
+      end
+    end
+    intervals = []
+    interval_start = nil
+    interval_end = nil
+    resp.each do |row|
+      if row.nil?
+        if !interval_start.nil? && !interval_end.nil?
+          # interval break and we know the start and end times
+          intervals += [[interval_start, 0], [interval_end, 0], nil]
+          interval_start = nil
+        end
+        next
+      end
+      if interval_start.nil?
+        interval_start = row[0]
+        next
+      end
+      interval_end = row[0]
+    end
+
+    if !interval_start.nil? && !interval_end.nil?
+      intervals += [[interval_start, 0], [interval_end, 0]]
+    end
+    elements.map do |e|
+      { id: e.id,
+        type: 'interval',
+        values: intervals }
+    end
+  end
+
+  def self.scale_value(value, element)
+    (value.to_f - element.offset) * element.scale_factor
+  end
+end
\ No newline at end of file

app/adapters/joule/adapter.rb

@@ -6,18 +6,18 @@ module Joule
     end
 
     def refresh(nilm)
-      db_service = UpdateDb.new(db: nilm.db)
+      db_service = UpdateDb.new(nilm.db)
       result = StubService.new
-      result.absorb_status(db_service.run(@backend.dbinfo, @backend.schema))
+      result.absorb_status(db_service.run(@backend.dbinfo, @backend.db_schema))
       module_service = UpdateModules.new(nilm)
-      result.absorb_status(module_service.run(@backend.module_info))
+      result.absorb_status(module_service.run(@backend.module_schemas))
       result
     end
 
     def refresh_stream(db_stream)
-      data = @backend.stream_info(db_stream)
-      service = UpdateStream.new(db_stream, data)
-      service.run
+      data = @backend.stream_info(db_stream.joule_id)
+      service = UpdateStream.new
+      service.run(db_stream, data[:stream], data[:data_info])
     end
 
     def save_stream(db_stream)
@@ -39,6 +39,11 @@ module Joule
           decimation_factor: data_service.decimation_factor
       }
     end
+
+    def module_interface(joule_module, req)
+      @backend.module_interface(joule_module, req)
+    end
+
     def node_type
       'joule'
     end

app/adapters/joule/backend.rb

@@ -2,24 +2,60 @@
 module Joule
   # Wrapper around Joule HTTP service
   class Backend
-   include HTTParty
-   default_timeout 5
-   open_timeout 5
-   read_timeout 5
+    include HTTParty
+    default_timeout 5
+    open_timeout 5
+    read_timeout 5
 
-   attr_reader :url
+    attr_reader :url
 
-   def initialize(url)
+    def initialize(url)
      @url = url
-   end
+    end
 
-   def module_info
+    def dbinfo
+     begin
+       resp = self.class.get("#{@url}/version")
+       return nil unless resp.success?
+       version = resp.parsed_response
+
+       resp = self.class.get("#{@url}/dbinfo")
+       return nil unless resp.success?
+       info = resp.parsed_response
+     rescue
+       return nil
+     end
+     # if the site exists but is not a nilm...
+     required_keys = %w(size other free reserved)
+     unless info.respond_to?(:has_key?) &&
+         required_keys.all? { |s| info.key? s }
+       return nil
+     end
+     {
+         version: version,
+         size_db: info['size'],
+         size_other: info['other'],
+         size_total: info['size'] + info['other'] + info['free'] + info['reserved']
+     }
+    end
+
+    def db_schema
+     begin
+       resp = self.class.get("#{@url}/streams.json")
+       return nil unless resp.success?
+     rescue
+       return nil
+     end
+     resp.parsed_response.deep_symbolize_keys
+    end
+
+    def module_schemas
      begin
        resp = self.class.get("#{@url}/modules.json")
        return nil unless resp.success?
        items = resp.parsed_response
        # if the site exists but is not a joule server...
-       required_keys = %w(name exec_cmd)
+       required_keys = %w(name inputs outputs)
        items.each do |item|
          return nil unless item.respond_to?(:has_key?) &&
                 required_keys.all? { |s| item.key? s }
@@ -28,11 +64,36 @@ module Joule
      rescue
        return nil
      end
-     return items
-   end
+     items
+    end
 
-   def module_interface(joule_module, req)
+    def module_interface(joule_module, req)
      self.class.get("#{@url}/interface/#{joule_module.joule_id}/#{req}")
-   end
+    end
+
+    def stream_info(joule_id)
+      begin
+       resp = self.class.get("#{@url}/stream.json?id=#{joule_id}")
+       return nil unless resp.success?
+      rescue
+       return nil
+      end
+      resp.parsed_response.deep_symbolize_keys
+    end
+
+    def load_data(joule_id, start_time, end_time, resolution)
+      options = { query: { "id": joule_id,
+                           "start": start_time,
+                           "end": end_time,
+                           "max-rows": resolution}}
+      begin
+        resp = self.class.get("#{@url}/data.json", options)
+        #TODO: handle interval data
+        return nil unless resp.success?
+      rescue
+        return nil
+      end
+      resp.parsed_response.symbolize_keys
+    end
   end
 end

app/adapters/joule/load_stream_data.rb

+# frozen_string_literal: true
+module Joule
+  # Loads stream data over the specified interval
+  class LoadStreamData
+    include ServiceStatus
+    attr_reader :data, :decimation_factor, :data_type
+
+
+    def initialize(backend)
+        super()
+        @backend = backend
+        @data = []
+        @data_type = 'unset' # interval, raw, decimated
+        @decimation_factor = 1
+      end
+
+    def run(db_stream, start_time, end_time, elements=[], resolution=nil)
+      # if elements are not explicitly passed, get all of them
+      if elements.empty?
+        elements = db_stream.db_elements.all.to_a
+      end
+      elements.sort_by!(&:column)
+
+      resolution = if resolution.nil?
+                     db_stream.db.max_points_per_plot
+                   else
+                     [db_stream.db.max_points_per_plot,resolution].min
+                   end
+      result = @backend.load_data(db_stream.joule_id, start_time, end_time, resolution)
+      # convert data into single array with nil's at interval boundaries
+      data = []
+      result[:data].each do |interval|
+        data += interval
+        data.push(nil)
+      end
+      if result[:decimated]
+        @data = DataBuilder.build_decimated_data(elements,data)
+        @data_type = 'decimated'
+      else
+        @data = DataBuilder.build_raw_data(elements,data)
+        @data_type = 'raw'
+      end
+      #TODO: handle interval data
+      @decimation_factor = 1 # TODO: fix this
+    end
+  end
+end
\ No newline at end of file

app/adapters/joule/update_db.rb

+# frozen_string_literal: true
+module Joule
+  # Handles construction of database objects
+  class UpdateDb
+    include ServiceStatus
+
+    def initialize(db)
+      @db = db
+      super()
+    end
+
+    def run(dbinfo, schema)
+      # check to make sure dbinfo and schema are set
+      # if either is nil, the database is not available
+      if dbinfo.nil? || schema.nil?
+        add_error("cannot contact database at #{@db.url}")
+        @db.update_attributes(available: false)
+        return self
+      end
+      # go through the schema and update the database
+      @db.root_folder ||= DbFolder.create(db: @db)
+      __update_folder(@db.root_folder, schema, '')
+      @db.available = true
+      @db.save
+      self
+    end
+
+    def __update_folder(db_folder, schema, parent_path)
+      attrs = schema.slice(*DbFolder.defined_attributes)
+      # add in extra attributes that require conversion
+      if db_folder.parent.nil?
+        attrs[:path] = ""
+      else
+        attrs[:path] = "#{parent_path}/#{schema[:name]}"
+      end
+      attrs[:joule_id] = schema[:id]
+      attrs[:hidden] = false
+      db_folder.update_attributes(attrs)
+      #puts db_folder.parent.id
+      # update or create subfolders
+      updated_ids = []
+      schema[:children].each do |child_schema|
+        child = db_folder.subfolders.find_by_joule_id(child_schema[:id])
+        child ||= DbFolder.new(parent: db_folder, db: db_folder.db)
+        __update_folder(child, child_schema, db_folder.path)
+        updated_ids << child_schema[:id]
+      end
+      # remove any subfolders that are no longer on the folder
+      db_folder.subfolders.where.not(joule_id: updated_ids).destroy_all
+
+      # update or create streams
+      updated_ids=[]
+      schema[:streams].each do |stream_schema|
+        stream = db_folder.db_streams.find_by_joule_id(stream_schema[:id])
+        stream ||= DbStream.new(db_folder: db_folder, db: db_folder.db)
+        __update_stream(stream, stream_schema, db_folder.path)
+        updated_ids << stream_schema[:id]
+      end
+      # remove any streams that are no longer in the folder
+      db_folder.db_streams.where.not(joule_id: updated_ids).destroy_all
+    end
+
+    def __update_stream(db_stream, schema, parent_path)
+      attrs = schema.slice(*DbStream.defined_attributes)
+      # add in extra attributes that require conversion
+      attrs[:path] = "#{parent_path}/#{schema[:name]}"
+      attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}"
+      attrs[:joule_id] = schema[:id]
+      attrs[:total_time] = 100 # non-zero TODO, fix load_element so we don't need this
+      db_stream.update_attributes(attrs)
+      db_stream.db_elements.destroy_all
+      schema[:elements].each do |element_config|
+        attrs = element_config.slice(*DbElement.defined_attributes)
+        # add in extra attributes that require conversion
+        attrs[:display_type] = element_config[:display_type].downcase
+        attrs[:column] = element_config[:index]
+        attrs[:plottable] = true
+        db_stream.db_elements << DbElement.new(attrs)
+      end
+    end
+
+
+  end
+end

app/adapters/joule/update_modules.rb

 # frozen_string_literal: true
 module Joule
   # Handles construction of database objects
-  class UpdateJouleModules
+  class UpdateModules
     include ServiceStatus
 
     def initialize(nilm)
@@ -9,47 +9,50 @@ module Joule
       @nilm = nilm
     end
 
-    def run(module_info)
+    def run(module_schemas)
       #module_info as returned by JouleBackend
-        if module_info.nil?
+        if module_schemas.nil?
           add_error("unable to retrieve module information")
           return self
         end
         #remove the previous modules
         @nilm.joule_modules.destroy_all
-        module_info.each do |info|
-          @nilm.joule_modules << _build_module(info)
+        module_schemas.each do |schema|
+          @nilm.joule_modules << _build_module(schema)
         end
 
         set_notice("refreshed modules")
         self
     end
 
-    def _build_module(info)
+    def _build_module(schema)
       # create JouleModule and associated pipes from
       # hash returned by the JouleAdapter.module_info
-      params = info.extract!(*JouleModule.joule_keys)
-      m = JouleModule.new(params)
+      attrs = schema.slice(*JouleModule.defined_attributes)
+      attrs[:pid] = schema[:statistics][:pid]
+      attrs[:web_interface] = schema[:has_interface]
+      attrs[:joule_id] = schema[:id]
+      m = JouleModule.create(attrs)
       # link inputs to database streams
-      info[:input_paths].each do |name, path|
+      schema[:inputs].each do |name, path|
         m.joule_pipes << JoulePipe.new(direction: 'input',
                                        name: name,
                                        db_stream: _retrieve_stream(path))
       end
-      info[:output_paths].each do |name, path|
+      schema[:outputs].each do |name, path|
         m.joule_pipes << JoulePipe.new(direction: 'output',
                                        name: name,
                                        db_stream: _retrieve_stream(path))
       end
-      return m
+      m
     end
 
     def _retrieve_stream(path)
-      dbStream = @nilm.db.db_streams.find_by_path(path)
-      if dbStream.nil?
+      db_stream = @nilm.db.db_streams.find_by_path(path)
+      if db_stream.nil?
         add_warning("[#{path}] not in database")
       end
-      dbStream
+      db_stream
     end
   end
 end

app/adapters/joule/update_stream.rb

+# frozen_string_literal: true
+module Joule
+  # Refresh a particular stream and load its data information
+  class UpdateStream
+    include ServiceStatus
+
+    def initialize
+      super()
+    end
+
+    def run(db_stream, schema, data_info)
+      attrs = schema.slice(*DbStream.defined_attributes)
+      # add in extra attributes that require conversion
+      attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}"
+      attrs[:joule_id] = schema[:id]
+      # add in data info
+      attrs[:start_time] = data_info[:start]
+      attrs[:end_time] = data_info[:end]
+      attrs[:total_time] = data_info[:end] - data_info[:start]
+      attrs[:total_rows] = data_info[:rows]
+      db_stream.update_attributes(attrs)
+      schema[:elements].each do |element_config|
+        attrs = element_config.slice(*DbElement.defined_attributes)
+        # add in extra attributes that require conversion
+        attrs[:display_type] = element_config[:display_type].downcase
+        attrs[:plottable] = true
+        elem = db_stream.db_elements.find_by_column(element_config[:index])
+        elem.update_attributes(attrs)
+      end
+      self
+    end
+  end
+end
\ No newline at end of file

app/adapters/nilmdb/adapter.rb

@@ -5,8 +5,8 @@ module Nilmdb
       @backend = Backend.new(url)
     end
 
-    def refresh(db:)
-      db_service = UpdateDb.new(db: db)
+    def refresh(nilm)
+      db_service = UpdateDb.new(nilm.db)
       db_service.run(@backend.dbinfo, @backend.schema)
     end
 

app/adapters/nilmdb/backend.rb

@@ -42,7 +42,7 @@ module Nilmdb
     def schema(path='')
       # GET extended info stream list
       begin
-        if(path.empty?)
+        if path.empty?
           resp = self.class.get("#{@url}/stream/list?extended=1")
         else
           resp = self.class.get("#{@url}/stream/list?path=#{path}&extended=1")
@@ -79,7 +79,7 @@ module Nilmdb
     end
 
     # return latest info about the specified stream
-    # TODO: the HTTP API does not
+    # the HTTP API does not
     # support wild cards so no decimations are returned
     #  {
     #    base_entry: ...,

app/adapters/nilmdb/load_stream_data.rb

@@ -38,7 +38,7 @@ module Nilmdb
     def run(db_stream, start_time, end_time, elements = [], resolution=nil)
 
       # if elements are not explicitly passed, get all of them
-      if(elements.empty?)
+      if elements.empty?
         elements = db_stream.db_elements.all.to_a
       end
       elements.sort_by!(&:column)

app/adapters/nilmdb/update_db.rb

@@ -4,7 +4,7 @@ module Nilmdb
   class UpdateDb
     include ServiceStatus
 
-    def initialize(db:)
+    def initialize(db)
       @db = db
       super()
     end

app/controllers/interfaces_controller.rb

 class InterfacesController < ActionController::Base
   before_action :authenticate_interface_user!, except: [:authenticate]
+  before_action :create_adapter, only: [:get, :put, :post, :delete]
+
   after_action :allow_wattsworth_iframe
   #GET /authenticate
   def authenticate
-    puts "here we go!"
     reset_session
     token = InterfaceAuthToken.find_by_value(params[:token])
     render :unauthorized and return if token.nil?
@@ -24,8 +25,7 @@ class InterfacesController < ActionController::Base
   def get
     path = params[:path] || ''
     req = path +"?"+request.query_string
-    backend = JouleAdapter.new(@joule_module.nilm.url)
-    render plain: backend.module_interface(@joule_module,req)
+    render plain: @node_adapter.module_interface(@joule_module,req)
   end
 
   def put
@@ -40,10 +40,10 @@ class InterfacesController < ActionController::Base
   private
 
   def authenticate_interface_user!
-    puts "trying to figure out the users..."
     @current_user = User.find_by_id(session[:user_id])
     @joule_module = JouleModule.find_by_id(session[:interface_id])
     render :unauthorized if (@current_user.nil? || @joule_module.nil?)
+
     #verify the session matches the URL
     #verify the user has permissions on this module
   end
@@ -54,4 +54,16 @@ class InterfacesController < ActionController::Base
     response.headers['X-Frame-Options'] = "ALLOW-FROM #{urls['frontend']}"
   end
 
+  def create_adapter
+    nilm = @joule_module.nilm
+    @node_adapter = NodeAdapterFactory.from_nilm(nilm)
+    if @node_adapter.nil?
+      @service = StubService.new
+      @service.add_error("Cannot contact installation")
+      render 'helpers/empty_response', status: :unprocessable_entity
+    end
+    if @node_adapter.node_type != 'joule'
+      render 'helpers/empty_response', status: :unprocessable_entity
+    end
+  end
 end

app/controllers/nilms_controller.rb

@@ -7,7 +7,8 @@ class NilmsController < ApplicationController
   before_action :authorize_viewer, only: [:show]
   before_action :authorize_owner, only: [:update, :refresh]
   before_action :authorize_admin, only: [:destroy]
-  before_action :create_adapter, only: [:create]
+  before_action :create_adapter_from_url, only: [:create]
+  before_action :create_adapter_from_nilm, only: [:show]
 
   # GET /nilms.json
   def index
@@ -20,7 +21,7 @@ class NilmsController < ApplicationController
     @role = current_user.get_nilm_permission(@nilm)
     #request new information from the NILM
     if params[:refresh]
-      @service = UpdateNilm.new(@adapter)
+      @service = UpdateNilm.new(@node_adapter)
       @service.run(@nilm)
       render status: @service.success? ? :ok : :unprocessable_entity
     else
@@ -75,7 +76,6 @@ class NilmsController < ApplicationController
     def set_nilm
       @nilm = Nilm.find(params[:id])
       @db = @nilm.db
-      @adapter = Nilmdb::Adapter.new(@nilm.url)
     end
 
     # Never trust parameters from the scary internet,
@@ -98,7 +98,7 @@ class NilmsController < ApplicationController
       head :unauthorized  unless current_user.views_nilm?(@nilm)
     end
 
-    def create_adapter
+    def create_adapter_from_url
       @node_adapter = NodeAdapterFactory.from_url(nilm_params[:url])
       if @node_adapter.nil?
         @service = StubService.new
@@ -106,4 +106,13 @@ class NilmsController < ApplicationController
         render 'helpers/empty_response', status: :unprocessable_entity
       end
     end
+
+    def create_adapter_from_nilm
+      @node_adapter = NodeAdapterFactory.from_nilm(@nilm)
+      if @node_adapter.nil?
+        @service = StubService.new
+        @service.add_error("Cannot contact installation")
+        render 'helpers/empty_response', status: :unprocessable_entity
+      end
+    end
 end

app/models/db_element.rb

@@ -28,6 +28,10 @@ class DbElement < ApplicationRecord
     self.display_type = 'continuous'
   end
 
+  def self.defined_attributes
+    [:name, :units, :default_min, :default_max, :scale_factor, :offset, :display_type, :column]
+  end
+
   # def name_path
   #  "#{db_stream.name_path}/#{self.name}"
   # end

app/models/db_stream.rb

@@ -28,8 +28,8 @@ class DbStream < ApplicationRecord
 
   validates_with DbDataTypeValidator
 
-  def defined_attributes
-     [:name, :name_abbrev, :description, :hidden]
+  def self.defined_attributes
+     [:name, :name_abbrev, :description, :hidden, :data_type]
   end
 
   # def name_path

app/models/joule_module.rb

@@ -3,13 +3,13 @@ class JouleModule < ApplicationRecord
   has_many :joule_pipes, dependent: :destroy
 
   # attributes accepted from the Joule json response
-  def self.joule_keys
+  def self.defined_attributes
     [:name, :description, :web_interface, :exec_cmd,
-     :status, :pid]
+     :status, :pid, :joule_id]
   end
   # attributes sent to the client
   def self.json_keys
     [:id, :name, :description, :web_interface, :exec_cmd,
-     :status, :pid]
+     :status, :pid, :joule_id]
   end
 end

app/services/data/load_element_data.rb

@@ -36,7 +36,11 @@ class LoadElementData
     #2 compute bounds by updating stream info if start/end are missing
     if start_time==nil || end_time==nil
       req_streams.map do |stream|
-        adapter = Nilmdb::Adapter.new(stream.db.url)
+        adapter = NodeAdapterFactory.from_nilm(stream.db.nilm)
+        if adapter.nil?
+          add_error("cannot contact installation")
+          return self
+        end
         adapter.refresh_stream(stream)
       end
     end
@@ -67,9 +71,8 @@ class LoadElementData
     combined_data = []
     req_streams.each do |stream|
       stream_elements = elements.select{|e| e.db_stream_id==stream.id}.to_a
-      adapter = Nilmdb::Adapter.new(stream.db.url)
+      adapter = NodeAdapterFactory.from_nilm(stream.db.nilm)
       result = adapter.load_data(stream, @start_time, @end_time,stream_elements,resolution)
-
       if not result.nil?
         combined_data.concat(result[:data])
       else

app/services/nilm/create_nilm.rb

@@ -35,7 +35,7 @@ class CreateNilm
     #give the owner 'admin' permissions on the nilm
     Permission.create(user: owner, nilm: nilm, role: 'admin')
     #update the database
-    msgs = @node_adapter.refresh(db: db)
+    msgs = @node_adapter.refresh(nilm)
     #errors on the database update are warnings on this service
     #because we can still add the NILM, it will just be offline
     add_warnings(msgs.errors + msgs.warnings)

app/services/nilm/update_nilm.rb

@@ -13,7 +13,7 @@ class UpdateNilm
       add_error('no associated db object')
       return self
     end
-    absorb_status(@node_adapter.refresh(db: nilm.db))
+    absorb_status(@node_adapter.refresh(nilm))
     self
   end
 end

config/environments/aws.rb

@@ -87,4 +87,9 @@ Rails.application.configure do
 
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
+
+  config.interface_url_template = lambda do |id|
+    return "http://#{id}.interfaces.wattsworth.net"
+  end
+
 end

config/environments/development.rb

@@ -66,4 +66,8 @@ Rails.application.configure do
         :methods =>  [:get, :post, :options, :delete, :put]
     end
   end
+  config.interface_url_template = lambda do |id|
+    return "http://#{id}.interfaces.wattsworth.local"
+  end
+
 end

config/environments/local.rb

@@ -84,4 +84,9 @@ Rails.application.configure do
 
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
+
+  config.interface_url_template = lambda do |id|
+    return "http://#{id}.interfaces.wattsworth.local"
+  end
+
 end

db/migrate/20180710014435_add_joule_info.rb

+class AddJouleInfo < ActiveRecord::Migration[5.2]
+  def change
+    add_column :nilms, :node_type, :string
+    add_column :db_folders, :joule_id, :integer
+    add_column :db_streams, :joule_id, :integer
+    add_index :db_streams, :joule_id
+    add_index :db_folders, :joule_id
+  end
+end

db/migrate/20180710014435_add_node_type_to_nilm.rb

-class AddNodeTypeToNilm < ActiveRecord::Migration[5.2]
-  def change
-    add_column :nilms, :node_type, :string, default: false, null: false
-  end
-end

db/schema.rb

@@ -69,6 +69,8 @@ ActiveRecord::Schema.define(version: 2018_07_10_014435) do
     t.integer "start_time", limit: 8
     t.integer "end_time", limit: 8
     t.integer "size_on_disk", limit: 8
+    t.integer "joule_id"
+    t.index ["joule_id"], name: "index_db_folders_on_joule_id"
   end
 
   create_table "db_streams", force: :cascade do |t|
@@ -88,6 +90,8 @@ ActiveRecord::Schema.define(version: 2018_07_10_014435) do
     t.boolean "hidden"
     t.integer "size_on_disk", limit: 8
     t.integer "db_id"
+    t.integer "joule_id"
+    t.index ["joule_id"], name: "index_db_streams_on_joule_id"
   end
 
   create_table "dbs", force: :cascade do |t|
@@ -155,7 +159,7 @@ ActiveRecord::Schema.define(version: 2018_07_10_014435) do
     t.string "url"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.string "node_type", default: "f", null: false
+    t.string "node_type"
   end
 
   create_table "permissions", force: :cascade do |t|

spec/adapters/joule/backend_spec.rb

@@ -4,11 +4,41 @@ require 'rails_helper'
 
 describe Joule::Backend do
   # use the benchtop server joule API
-  let (:url) {'http://172.16.1.12/joule'}
-  it 'retrieves module infos', :vcr do
+  let (:url) {'http://nuc:8088'}
+  it 'retrieves database schema', :vcr do
     backend = Joule::Backend.new(url)
-    backend.module_info.each do |m|
-      expect(m).to include(:name, :exec_cmd, :web_interface)
+    schema = backend.db_schema
+    # make sure keys are symbolized
+    expect(schema).to include(:name, :id, :streams, :children)
+    # should be a tree structure
+    expect(schema[:children][0]).to include(:name, :id, :streams, :children)
+  end
+  it 'retrieves module schema', :vcr do
+    backend = Joule::Backend.new(url)
+    backend.module_schemas.each do |m|
+      expect(m).to include(:name, :inputs, :outputs)
     end
   end
+
+  it 'loads raw data', :vcr do
+    backend = Joule::Backend.new(url)
+    resp = backend.load_data(6,
+                             1531248642561047,
+                             1531248642581047,
+                             200)
+    expect(resp[:decimated]).to be false
+    expect(resp[:data].count).to be > 0
+    expect(resp[:data].count).to be < 200
+  end
+
+  it 'loads decimated data', :vcr do
+    backend = Joule::Backend.new(url)
+    resp = backend.load_data(6,
+                             1531248642561047,
+                             1531330705273202,
+                             20)
+    expect(resp[:decimated]).to be true
+    expect(resp[:data].count).to be > 0
+    expect(resp[:data].count).to be < 200
+  end
 end

spec/adapters/joule/test_db_schema.json

+{
+  "id": 1,
+  "name": "root",
+  "description": null,
+  "children": [
+    {
+      "id": 2,
+      "name": "folder_2",
+      "description": null,
+      "children": [
+
+      ],
+      "streams": [
+        {
+          "id": 1,
+          "name": "stream_2_1",
+          "description": "",
+          "datatype": "INT16",
+          "keep_us": -1,
+          "decimate": true,
+          "elements": [
+            {
+              "id": 12,
+              "index": 0,
+              "name": "top",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            },
+            {
+              "id": 13,
+              "index": 1,
+              "name": "bottom",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "name": "folder_1",
+      "description": null,
+      "children": [
+
+      ],
+      "streams": [
+        {
+          "id": 2,
+          "name": "stream_1_2",
+          "description": "",
+          "datatype": "UINT8",
+          "keep_us": -1,
+          "decimate": true,
+          "elements": [
+            {
+              "id": 14,
+              "index": 0,
+              "name": "a",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            },
+            {
+              "id": 15,
+              "index": 1,
+              "name": "b",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            },
+            {
+              "id": 16,
+              "index": 2,
+              "name": "c",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "name": "stream_1_1",
+          "description": "",
+          "datatype": "FLOAT32",
+          "keep_us": -1,
+          "decimate": true,
+          "elements": [
+            {
+              "id": 17,
+              "index": 0,
+              "name": "x",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": 100,
+              "default_min": null
+            },
+            {
+              "id": 18,
+              "index": 1,
+              "name": "y",
+              "units": null,
+              "plottable": true,
+              "display_type": "EVENT",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": -6
+            },
+            {
+              "id": 19,
+              "index": 2,
+              "name": "z",
+              "units": "watts",
+              "plottable": true,
+              "display_type": "DISCRETE",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "name": "folder_3",
+      "description": null,
+      "children": [
+        {
+          "id": 5,
+          "name": "folder_3_1",
+          "description": null,
+          "children": [
+
+          ],
+          "streams": [
+            {
+              "id": 4,
+              "name": "stream_3_1_1",
+              "description": "",
+              "datatype": "INT32",
+              "keep_us": -1,
+              "decimate": true,
+              "elements": [
+                {
+                  "id": 20,
+                  "index": 0,
+                  "name": "a",
+                  "units": null,
+                  "plottable": true,
+                  "display_type": "CONTINUOUS",
+                  "offset": 0,
+                  "scale_factor": 1,
+                  "default_max": null,
+                  "default_min": null
+                },
+                {
+                  "id": 21,
+                  "index": 1,
+                  "name": "b",
+                  "units": null,
+                  "plottable": true,
+                  "display_type": "CONTINUOUS",
+                  "offset": 0,
+                  "scale_factor": 1,
+                  "default_max": null,
+                  "default_min": null
+                },
+                {
+                  "id": 22,
+                  "index": 2,
+                  "name": "c",
+                  "units": null,
+                  "plottable": true,
+                  "display_type": "CONTINUOUS",
+                  "offset": 0,
+                  "scale_factor": 1,
+                  "default_max": null,
+                  "default_min": null
+                }
+              ]
+            }
+          ]
+        }
+      ],
+      "streams": [
+        {
+          "id": 5,
+          "name": "stream_3_1",
+          "description": "",
+          "datatype": "UINT16",
+          "keep_us": -1,
+          "decimate": true,
+          "elements": [
+            {
+              "id": 23,
+              "index": 0,
+              "name": "a",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            },
+            {
+              "id": 24,
+              "index": 1,
+              "name": "b",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            },
+            {
+              "id": 25,
+              "index": 2,
+              "name": "c",
+              "units": null,
+              "plottable": true,
+              "display_type": "CONTINUOUS",
+              "offset": 0,
+              "scale_factor": 1,
+              "default_max": null,
+              "default_min": null
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "name": "folder_4",
+      "description": null,
+      "children": [
+        {
+          "id": 7,
+          "name": "folder_4_1",
+          "description": null,
+          "children": [
+
+          ],
+          "streams": [
+
+          ]
+        }
+      ],
+      "streams": [
+
+      ]
+    }
+  ],
+  "streams": [
+
+  ]
+}
\ No newline at end of file

spec/adapters/joule/test_module_schema.json

+[
+  {
+    "name": "Module3",
+    "description": "a filter",
+    "has_interface": true,
+    "inputs": {
+      "input1": "/folder_1/stream_1_1",
+      "input2": "/folder_1/stream_1_2"
+    },
+    "outputs": {
+      "output": "/folder_2/stream_2_1"
+    },
+    "statistics": {
+      "pid": 29702,
+      "create_time": 1531319112.03,
+      "cpu_percent": 0,
+      "memory": 8286208
+    }
+  },
+  {
+    "name": "Module2",
+    "description": "a reader",
+    "has_interface": false,
+    "inputs": {},
+    "outputs": {
+      "output": "/folder_1/stream_1_2"
+    },
+    "statistics": {
+      "pid": 29703,
+      "create_time": 1531319112.05,
+      "cpu_percent": 0,
+      "memory": 8364032
+    }
+  },
+  {
+    "name": "Module1",
+    "description": "a reader",
+    "has_interface": true,
+    "inputs": {},
+    "outputs": {
+      "output": "/folder_1/stream_1_1"
+    },
+    "statistics": {
+      "pid": 29704,
+      "create_time": 1531319112.07,
+      "cpu_percent": 0,
+      "memory": 8224768
+    }
+  },
+  {
+    "name": "Module4",
+    "description": "a filter",
+    "has_interface": false,
+    "inputs": {
+      "input1": "/folder_1/stream_1_1",
+      "input2": "/folder_2/stream_2_1"
+    },
+    "outputs": {
+      "output1": "/folder_3/stream_3_1",
+      "output2": "/folder_3/folder_3_1/stream_3_1_1"
+    },
+    "statistics": {
+      "pid": 29705,
+      "create_time": 1531319112.1,
+      "cpu_percent": 0,
+      "memory": 8515584
+    }
+  }
+]
\ No newline at end of file

spec/adapters/joule/update_db_spec.rb

+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'json'
+
+# Test Database Schema:
+# │
+# ├── folder_1
+# │   ├── stream_1_1: float32_3
+# │   └── stream_1_2: uint8_3
+# ├── folder_2
+# │   └── stream_2_1: int16_2
+# ├── folder_3
+# │   ├── folder_3_1
+# │   │   └── stream_3_1_1: int32_3
+# │   └── stream_3_1: uint16_3
+# └── folder_4
+#     └── folder_4_1
+
+describe Joule::UpdateDb do
+  before do
+    raw = File.read(File.dirname(__FILE__)+"/test_db_schema.json")
+    @schema = JSON.parse(raw).deep_symbolize_keys
+    @db = Db.new
+  end
+
+  let(:dbinfo) { {} }
+  describe '*run*' do
+    describe 'given the test database schema' do
+      it 'builds the database' do
+        service = Joule::UpdateDb.new(@db)
+        service.run({}, @schema)
+        expect(@db.root_folder.subfolders.count).to eq 4
+        # go through Folder 1 carefully
+        folder_1 = @db.root_folder.subfolders.where(name: 'folder_1').first
+        expect(folder_1.subfolders.count).to eq 0
+        expect(folder_1.db_streams.count).to eq 2
+        expect(folder_1.path).to eq '/folder_1'
+        stream_1_1 = folder_1.db_streams.where(name: 'stream_1_1').first
+        expect(stream_1_1.data_type).to eq 'float32_3'
+        expect(stream_1_1.path).to eq '/folder_1/stream_1_1'
+        expect(stream_1_1.db_elements.count).to eq 3
+        x = stream_1_1.db_elements.where(name: 'x').first
+        expect(x.display_type).to eq 'continuous'
+        expect(x.column).to eq 0
+        expect(x.default_max).to eq 100
+        y = stream_1_1.db_elements.where(name: 'y').first
+        expect(y.display_type).to eq 'event'
+        expect(y.column).to eq 1
+        expect(y.default_min).to eq -6
+        z = stream_1_1.db_elements.where(name: 'z').first
+        expect(z.display_type).to eq 'discrete'
+        expect(z.column).to eq 2
+        expect(z.units).to eq "watts"
+
+        # quick checks
+        expect(DbElement.count).to eq 14
+        expect(DbStream.count).to eq 5
+        expect(DbFolder.count).to eq 7
+      end
+    end
+  end
+end
\ No newline at end of file

spec/adapters/joule/update_modules_spec.rb

@@ -2,50 +2,59 @@
 
 require 'rails_helper'
 
-describe 'UpdateJouleModules' do
+describe Joule::UpdateModules do
 
+  before do
+    raw = File.read(File.dirname(__FILE__)+"/test_module_schema.json")
+    @schema = JSON.parse(raw).map{|item| item.deep_symbolize_keys}
+  end
   it 'replaces existing modules with new ones' do
     nilm = create(:nilm)
     nilm.joule_modules << create(:joule_module, name: 'prev1')
     nilm.joule_modules << create(:joule_module, name: 'prev2')
-    backend = MockJouleAdapter.new
-    backend.add_module("new1",inputs={i1: '/path/1'},
-                              outputs={o1: '/path/2'})
-    backend.add_module("new2",inputs={i1: '/path/3',i2: '/path/4'},
-                             outputs={o1: '/path/5',o2: '/path/5'})
     service = Joule::UpdateModules.new(nilm)
-    service.run(backend.module_info)
+    service.run(@schema)
     expect(service.success?).to be true
     # new modules are in the database
-    expect(nilm.joule_modules.find_by_name('new1')).to be_present
-    expect(nilm.joule_modules.find_by_name('new2')).to be_present
+    %w(Module1 Module2 Module3 Module4).each do |name|
+      expect(nilm.joule_modules.find_by_name(name)).to be_present
+    end
     # old ones are gone
-    expect(JouleModule.count).to eq 2
+    expect(JouleModule.count).to eq 4
     # pipes are updated as well
-    n1 = nilm.joule_modules.find_by_name('new1')
-    expect(n1.joule_pipes.count).to eq 2
-    n2 = nilm.joule_modules.find_by_name('new2')
-    expect(n2.joule_pipes.count).to eq 4
+    m2 = nilm.joule_modules.find_by_name('Module2')
+    expect(m2.joule_pipes.count).to eq 1
+    m3 = nilm.joule_modules.find_by_name('Module3')
+    expect(m3.joule_pipes.count).to eq 3
+    # web interface status is correct
+    expect(m2.web_interface).to be false
+    expect(m3.web_interface).to be true
     # old pipes are gone
-    expect(JoulePipe.count).to eq 6
+    expect(JoulePipe.count).to eq 9
   end
   it 'produces a warning if a stream is not in the database' do
     nilm = create(:nilm)
-    backend = MockJouleAdapter.new
-    backend.add_module("module",outputs={output: '/missing/path'})
     service = Joule::UpdateModules.new(nilm)
-    service.run(backend.module_info)
+    service.run(@schema)
     expect(service.warnings?).to be true
   end
   it 'links db_stream to the pipe if the stream is in the database' do
     nilm = create(:nilm)
-    nilm.db.db_streams << create(:db_stream, path: '/matched/path1')
-    nilm.db.db_streams << create(:db_stream, path: '/matched/path2')
-    backend = MockJouleAdapter.new
-    backend.add_module("module",inputs={input: '/matched/path1'},
-                                outputs={output: '/matched/path2'})
+    # create streams for pipe connections
+    nilm.db.db_streams << create(:db_stream, path: '/folder_1/stream_1_1')
+    nilm.db.db_streams << create(:db_stream, path: '/folder_1/stream_1_2')
+    nilm.db.db_streams << create(:db_stream, path: '/folder_2/stream_2_1')
     service = Joule::UpdateModules.new(nilm)
-    service.run(backend.module_info)
+    #just run module3
+    service.run([@schema[0]])
+    # make sure pipes are connected
+    m3 = nilm.joule_modules.find_by_name('Module3')
+    pipe = m3.joule_pipes.where(direction: 'input', name: 'input1').first
+    expect(pipe.db_stream.path).to eq('/folder_1/stream_1_1')
+    pipe = m3.joule_pipes.where(direction: 'input', name: 'input2').first
+    expect(pipe.db_stream.path).to eq('/folder_1/stream_1_2')
+    pipe = m3.joule_pipes.where(direction: 'output', name: 'output').first
+    expect(pipe.db_stream.path).to eq('/folder_2/stream_2_1')
     expect(service.warnings?).to be false
   end
 end

spec/adapters/nilmdb/update_db_spec.rb

@@ -28,8 +28,7 @@ describe 'UpdateDb' do
   describe '*run*' do
     def update_with_schema(schema, db: nil)
       @db = db || Db.new
-      @service = Nilmdb::UpdateDb.new(db: @db)
-      mock_info =
+      @service = Nilmdb::UpdateDb.new(@db)
       @service.run(dbinfo, schema) #ignore dbinfo
       @root = @db.root_folder
     end

spec/adapters/nilmdb/update_folder_spec.rb

@@ -4,7 +4,7 @@ require 'rails_helper'
 
 describe 'UpdateFolder service' do
   let(:db) { Db.new }
-  let(:service) { Nilmdb::UpdateDb.new(db: db) }
+  let(:service) { Nilmdb::UpdateDb.new(db) }
   let(:helper) { DbSchemaHelper.new }
   let(:mock_dbinfo) { {} }
 
@@ -20,7 +20,7 @@ describe 'UpdateFolder service' do
     folder = DbFolder.find_by_name('old_name')
     expect(folder).to be_present
     # run update again with new metadata
-    service = Nilmdb::UpdateDb.new(db: db)
+    service = Nilmdb::UpdateDb.new(db)
     service.run(mock_dbinfo, [helper.entry('/folder1/subfolder/info',
                                            metadata: { name: 'new_name' })])
     folder.reload

spec/adapters/nilmdb/update_stream_spec.rb

@@ -4,7 +4,7 @@ require 'rails_helper'
 
 describe 'UpdateStream service' do
   let(:db) { Db.new }
-  let(:service) { Nilmdb::UpdateDb.new(db: db) }
+  let(:service) { Nilmdb::UpdateDb.new(db) }
   let(:helper) { DbSchemaHelper.new }
   let(:mock_dbinfo) { {} }
 
@@ -21,7 +21,7 @@ describe 'UpdateStream service' do
     stream = DbStream.find_by_name('old_name')
     expect(stream).to be_present
     # run update again with new metadata
-    service = Nilmdb::UpdateDb.new(db: db)
+    service = Nilmdb::UpdateDb.new(db)
     service.run(mock_dbinfo, [helper.entry('/folder1/stream1',
                               metadata: { name: 'new_name' })])
     stream.reload
@@ -59,7 +59,7 @@ describe 'UpdateStream service' do
     expect(element).to be_present
     # run update again with new metadata
     schema[0][:elements][0][:name] = 'new_name'
-    service = Nilmdb::UpdateDb.new(db: db)
+    service = Nilmdb::UpdateDb.new(db)
     service.run(mock_dbinfo, schema)
     element.reload
     expect(element.name).to eq('new_name')

spec/models/interface_auth_token_spec.rb

-require 'rails_helper'
-
-RSpec.describe InterfaceAuthToken, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
-end

spec/services/data/load_element_data_spec.rb

@@ -17,7 +17,7 @@ RSpec.describe 'LoadElementData' do
                       {id: @elem2.id, values: 'mock2'}]
       @mock_adapter = MockAdapter.new([stream: @db_stream,
                                        data: @stream_data])
-      allow(Nilmdb::Adapter).to receive(:new).and_return(@mock_adapter)
+      allow(NodeAdapterFactory).to receive(:from_nilm).and_return(@mock_adapter)
     end
     it 'makes one request for the stream data' do
       #expect(@mock_adapter).to receive(:load_data)
@@ -44,7 +44,7 @@ RSpec.describe 'LoadElementData' do
       @mock_adapter = MockAdapter.new(
         [{stream: @db_stream1, data: @stream1_data},
          {stream: @db_stream2, data: @stream2_data}])
-      allow(Nilmdb::Adapter).to receive(:new).and_return(@mock_adapter)
+      allow(NodeAdapterFactory).to receive(:from_nilm).and_return(@mock_adapter)
 
     end
     it 'makes one request per stream' do
@@ -77,7 +77,7 @@ RSpec.describe 'LoadElementData' do
       @mock_adapter = MockAdapter.new(
         [{stream: @db_stream1, data: @stream1_data},
          {stream: @db_stream2, data: nil}])
-      allow(Nilmdb::Adapter).to receive(:new).and_return(@mock_adapter)
+      allow(NodeAdapterFactory).to receive(:from_nilm).and_return(@mock_adapter)
     end
     it 'fills in the data that is available' do
       service = LoadElementData.new