added nilm_id to json for db_streams

app/adapters/db_adapter.rb

@@ -3,7 +3,10 @@
 # Wrapper around NilmDB HTTP service
 # Wrapper around NilmDB HTTP service
 class DbAdapter
 class DbAdapter
   include HTTParty
   include HTTParty
-  default_timeout 10
+  default_timeout 5
+  open_timeout 5
+  read_timeout 5
+
   attr_reader :url
   attr_reader :url
 
 
   def initialize(url)
   def initialize(url)

app/controllers/db_elements_controller.rb

@@ -3,16 +3,7 @@ class DbElementsController < ApplicationController
   before_action :authenticate_user!
   before_action :authenticate_user!
 
 
 
 
-  #def index
-  #  @elements = DbElement.find(JSON.parse(params[:elements]))
-  #  # make sure the user is allowed to view these elements
-  #  @elements.each do |elem|
-  #    unless current_user.views_nilm?(elem.db_stream.db.nilm)
-  #      head :unauthorized
-  #      return
-  #    end
-  #  end
-  #end
+
 
 
   def data
   def data
     req_elements = DbElement.find(JSON.parse(params[:elements]))
     req_elements = DbElement.find(JSON.parse(params[:elements]))
@@ -26,12 +17,28 @@ class DbElementsController < ApplicationController
     # make sure the time range makes sense
     # make sure the time range makes sense
     start_time = (params[:start_time].to_i unless params[:start_time].nil?)
     start_time = (params[:start_time].to_i unless params[:start_time].nil?)
     end_time = (params[:end_time].to_i unless params[:end_time].nil?)
     end_time = (params[:end_time].to_i unless params[:end_time].nil?)
+    #requested resolution (leave blank for max possible)
+    resolution = (params[:resolution].to_i unless params[:resolution].nil?)
+    # padding: percentage of data to retrieve beyond start|end
+    padding = params[:padding].nil? ? 0 : params[:padding].to_f
+
 
 
     # retrieve the data for the requested elements
     # retrieve the data for the requested elements
     @service = LoadElementData.new
     @service = LoadElementData.new
-    @service.run(req_elements, start_time, end_time)
-    @start_time = @service.start_time
-    @end_time = @service.end_time
+
+    #if start and end are specified, calculate padding
+    if !start_time.nil? && !end_time.nil?
+      actual_start = (start_time - (end_time-start_time)*padding).to_i
+      actual_end   = (end_time   + (end_time-start_time)*padding).to_i
+      @service.run(req_elements, actual_start, actual_end, resolution)
+      @start_time = start_time
+      @end_time = end_time
+    #otherwise let the service determine the start/end automatically
+    else
+      @service.run(req_elements, start_time, end_time, resolution)
+      @start_time = @service.start_time
+      @end_time = @service.end_time
+    end
 
 
     render status: @service.success? ? :ok : :unprocessable_entity
     render status: @service.success? ? :ok : :unprocessable_entity
   end
   end

app/controllers/db_streams_controller.rb

@@ -7,6 +7,22 @@ class DbStreamsController < ApplicationController
   before_action :authorize_viewer, only: [:data]
   before_action :authorize_viewer, only: [:data]
   before_action :authorize_owner, only: [:update]
   before_action :authorize_owner, only: [:update]
 
 
+  def index
+    if params[:streams].nil?
+      head :unprocessable_entity
+      return
+    end
+
+    @streams = DbStream.find(JSON.parse(params[:streams]))
+    # make sure the user is allowed to view these streams
+    @streams.each do |stream|
+      unless current_user.views_nilm?(stream.db.nilm)
+        head :unauthorized
+        return
+      end
+    end
+  end
+
   def update
   def update
     adapter = DbAdapter.new(@db.url)
     adapter = DbAdapter.new(@db.url)
     @service = EditStream.new(adapter)
     @service = EditStream.new(adapter)

app/services/data/load_element_data.rb

@@ -17,6 +17,8 @@ class LoadElementData
   # start_time and end_time are unix us
   # start_time and end_time are unix us
   # if start_time is nil it is set to earliest timestamp
   # if start_time is nil it is set to earliest timestamp
   # if end_time is nil it is set to latest timestamp
   # if end_time is nil it is set to latest timestamp
+  # if resolution is nil, retrieve highest resolution possible
+
   #
   #
   # sets data
   # sets data
   # data:
   # data:
@@ -24,7 +26,7 @@ class LoadElementData
   #    {id: element_id, values: [...]},...]
   #    {id: element_id, values: [...]},...]
   # see load_stream_data for details on the value structure
   # see load_stream_data for details on the value structure
   #
   #
-  def run(elements, start_time, end_time)
+  def run(elements, start_time, end_time, resolution = nil)
     #1 figure out what streams need to be pulled
     #1 figure out what streams need to be pulled
     req_streams = []
     req_streams = []
     elements.each do |elem|
     elements.each do |elem|
@@ -76,7 +78,7 @@ class LoadElementData
       adapter = DbAdapter.new(stream.db.url)
       adapter = DbAdapter.new(stream.db.url)
       data_service = LoadStreamData.new(adapter)
       data_service = LoadStreamData.new(adapter)
       stream_elements = elements.select{|e| e.db_stream_id==stream.id}.to_a
       stream_elements = elements.select{|e| e.db_stream_id==stream.id}.to_a
-      data_service.run(stream, @start_time, @end_time,stream_elements)
+      data_service.run(stream, @start_time, @end_time,stream_elements,resolution)
 
 
       if data_service.success?
       if data_service.success?
         combined_data.concat(data_service.data)
         combined_data.concat(data_service.data)

app/services/data/load_stream_data.rb

@@ -17,6 +17,8 @@ class LoadStreamData
   # associated database, sets data and data_type
   # associated database, sets data and data_type
   # specify a subset of elements as an optional array
   # specify a subset of elements as an optional array
   # if ommitted, all elements are extracted from the stream (expensive!)
   # if ommitted, all elements are extracted from the stream (expensive!)
+  # optionally specify a resolution, if omitted, returns maximum resolution
+  # allowed by the nilm
   #
   #
   # sets data and data_type
   # sets data and data_type
   # data_type: raw
   # data_type: raw
@@ -33,7 +35,7 @@ class LoadStreamData
   # data:
   # data:
   #   [{id: element_id, type: decimated, values: [[start,0],[end,0],nil,...]}]
   #   [{id: element_id, type: decimated, values: [[start,0],[end,0],nil,...]}]
   #
   #
-  def run(db_stream, start_time, end_time, elements = [])
+  def run(db_stream, start_time, end_time, elements = [], resolution=nil)
 
 
     # if elements are not explicitly passed, get all of them
     # if elements are not explicitly passed, get all of them
     if(elements.empty?)
     if(elements.empty?)
@@ -41,7 +43,11 @@ class LoadStreamData
     end
     end
     elements.sort_by!(&:column)
     elements.sort_by!(&:column)
 
 
-    resolution = db_stream.db.max_points_per_plot
+    resolution = if resolution.nil?
+                   db_stream.db.max_points_per_plot
+                 else
+                   [db_stream.db.max_points_per_plot,resolution].min
+                 end
     valid_decim = findValidDecimationLevel(db_stream, start_time)
     valid_decim = findValidDecimationLevel(db_stream, start_time)
     # valid_decim is the highest resolution, find one we can plot
     # valid_decim is the highest resolution, find one we can plot
     plottable_decim = findPlottableDecimationLevel(
     plottable_decim = findPlottableDecimationLevel(
@@ -98,9 +104,8 @@ class LoadStreamData
   # the decimation level will be 0
   # the decimation level will be 0
   #
   #
   def findPlottableDecimationLevel(
   def findPlottableDecimationLevel(
-    db_stream, valid_decim, start_time, end_time, _resolution
+    db_stream, valid_decim, start_time, end_time, resolution
   )
   )
-
     path = db_stream.path
     path = db_stream.path
     path += "~decim-#{valid_decim.level}" if valid_decim.level > 1
     path += "~decim-#{valid_decim.level}" if valid_decim.level > 1
     # figure out how much data this stream has over the interval
     # figure out how much data this stream has over the interval
@@ -112,16 +117,15 @@ class LoadStreamData
     # find out how much raw data exists over the specified interval
     # find out how much raw data exists over the specified interval
     raw_count = count * valid_decim.level
     raw_count = count * valid_decim.level
     # now we can find the right decimation level for plotting
     # now we can find the right decimation level for plotting
-    max_count = db_stream.db.max_points_per_plot
     # if the valid decim can be plotted, use it
     # if the valid decim can be plotted, use it
-    return valid_decim if raw_count <= max_count
+    return valid_decim if raw_count <= resolution
     # otherwise look for a higher decimation level
     # otherwise look for a higher decimation level
     found_valid_decim = false
     found_valid_decim = false
     db_stream.db_decimations
     db_stream.db_decimations
              .where('level >= ?', valid_decim.level)
              .where('level >= ?', valid_decim.level)
              .order(:level)
              .order(:level)
              .each do |decim|
              .each do |decim|
-      if raw_count / decim.level <= max_count
+      if raw_count / decim.level <= resolution
         # the lowest decimation level is the best
         # the lowest decimation level is the best
         return decim
         return decim
       end
       end

app/views/db_folders/_db_folder.json.jbuilder

@@ -6,6 +6,7 @@ end
 
 
 json.streams(db_folder.db_streams.includes(:db_elements)) do |stream|
 json.streams(db_folder.db_streams.includes(:db_elements)) do |stream|
   json.extract! stream, *DbStream.json_keys
   json.extract! stream, *DbStream.json_keys
+  json.nilm_id nilm.id
   json.elements(stream.db_elements) do |element|
   json.elements(stream.db_elements) do |element|
     json.extract! element, *DbElement.json_keys
     json.extract! element, *DbElement.json_keys
   end
   end

app/views/db_folders/show.json.jbuilder

 # frozen_string_literal: true
 # frozen_string_literal: true
 json.partial! 'db_folders/db_folder',
 json.partial! 'db_folders/db_folder',
-              db_folder: @db_folder
+              db_folder: @db_folder, nilm: @nilm

app/views/db_streams/_db_stream.json.jbuilder

 
 
 # frozen_string_literal: true
 # frozen_string_literal: true
 json.extract! db_stream, *DbStream.json_keys
 json.extract! db_stream, *DbStream.json_keys
+json.nilm_id db_stream.db.nilm.id
 
 
 json.elements(db_stream.db_elements) do |element|
 json.elements(db_stream.db_elements) do |element|
   json.extract! element, *DbElement.json_keys
   json.extract! element, *DbElement.json_keys

app/views/db_streams/data.csv.erb

 ###############################################
 ###############################################
 # Stream: <%=@db_stream.name%>
 # Stream: <%=@db_stream.name%>
+# Installation: <%=@nilm.name%> <%unless @nilm.description.blank?%>(<%=@nilm.description%>)<%end%>
 # Path:   <%=@db_stream.path%>
 # Path:   <%=@db_stream.path%>
-# Source: <%=@nilm.name%> <%unless @nilm.description.blank?%>(<%=@nilm.description%>)<%end%>
-#
+# URL: <%=@nilm.url%>
 #
 #
 # start:             <%=Time.at(@legend[:start_time]/1e6)%>
 # start:             <%=Time.at(@legend[:start_time]/1e6)%>
 # end:               <%=Time.at(@legend[:end_time]/1e6)%>
 # end:               <%=Time.at(@legend[:end_time]/1e6)%>

app/views/db_streams/index.json.jbuilder

+json.array! @streams do |stream|
+  json.extract! stream, *DbStream.json_keys
+  json.nilm_id stream.db.nilm.id
+  json.elements(stream.db_elements) do |element|
+    json.extract! element, *DbElement.json_keys
+  end
+end

app/views/db_streams/show.json.jbuilder

-# frozen_string_literal: true
-json.partial! 'db_streams/db_stream',
-              db_folder: @db_stream

config/routes.rb

@@ -12,7 +12,7 @@ Rails.application.routes.draw do
   end
   end
   resources :dbs, only: [:show, :update]
   resources :dbs, only: [:show, :update]
   resources :db_folders, only: [:show, :update]
   resources :db_folders, only: [:show, :update]
-  resources :db_streams, only: [:update] do
+  resources :db_streams, only: [:index, :update] do
     member do
     member do
       post 'data'
       post 'data'
     end
     end

spec/controllers/db_elements_controller_spec.rb

@@ -18,8 +18,6 @@ RSpec.describe DbElementsController, type: :request do
         stream.db_elements << @elem2
         stream.db_elements << @elem2
       end
       end
       it "returns elements with data" do
       it "returns elements with data" do
-
-
         @service_data = [{ id: @elem1.id, data: 'mock1' },
         @service_data = [{ id: @elem1.id, data: 'mock1' },
                          { id: @elem2.id, data: 'mock2' }]
                          { id: @elem2.id, data: 'mock2' }]
         @mock_service = instance_double(LoadElementData,
         @mock_service = instance_double(LoadElementData,
@@ -40,6 +38,31 @@ RSpec.describe DbElementsController, type: :request do
         body = JSON.parse(response.body)
         body = JSON.parse(response.body)
         expect(body['data'].count).to eq(2)
         expect(body['data'].count).to eq(2)
       end
       end
+
+      it 'computes padding if specified' do
+        @service_data = [{ id: @elem1.id, data: 'mock1' },
+                         { id: @elem2.id, data: 'mock2' }]
+        @mock_service = instance_double(LoadElementData,
+                                        run: StubService.new,
+                                        success?: true, notices: [], warnings: [], errors: [],
+                                        data: @service_data)
+        allow(LoadElementData).to receive(:new).and_return(@mock_service)
+        expect(@mock_service).to receive(:run).with([@elem1,@elem2],90,210,nil)
+        @auth_headers = user1.create_new_auth_token
+        get '/db_elements/data.json',
+            params: { elements: [@elem1.id, @elem2.id].to_json,
+                      start_time: 100, end_time: 200, padding: 0.1 },
+            headers: @auth_headers
+        expect(response).to have_http_status(:ok)
+        # check to make sure JSON renders the elements
+        body = JSON.parse(response.body)
+        expect(body['data'].count).to eq(2)
+        # reported time bounds should *NOT* include padding
+        body['data'].map do |data|
+          expect(data['start_time']).to eq 100
+          expect(data['end_time']).to eq 200
+        end
+      end
       it 'returns error if time bounds are invalid' do
       it 'returns error if time bounds are invalid' do
         @auth_headers = user1.create_new_auth_token
         @auth_headers = user1.create_new_auth_token
         get '/db_elements/data.json',
         get '/db_elements/data.json',

spec/controllers/db_streams_controller_spec.rb

@@ -12,7 +12,52 @@ RSpec.describe DbStreamsController, type: :request do
                                  db: john_nilm.db)
                                  db: john_nilm.db)
   end
   end
 
 
-  # index action does not exist
+  describe 'GET index' do
+    let(:viewer) {create(:user)}
+    let(:nilm) {create(:nilm, viewers: [viewer])}
+    let(:db) {create(:db, nilm: nilm)}
+    let(:stream2) {create(:db_stream, db: db)}
+    let(:stream1) {create(:db_stream, db: db)}
+    let(:other_nilm) {create(:nilm)}
+    let(:other_db) {create(:db, nilm: other_nilm)}
+    let(:other_stream) {create(:db_stream, db: other_db)}
+
+    context 'with viewer permissions' do
+      it 'returns array of requested streams' do
+        @auth_headers = viewer.create_new_auth_token
+        get "/db_streams.json",
+          params: {streams: [stream1.id, stream2.id].to_json},
+          headers: @auth_headers
+        expect(response).to have_http_status(:ok)
+        # check to make sure JSON renders the streams
+        streams = JSON.parse(response.body)
+        expect(streams.count).to eq(2)
+      end
+      it 'returns unauthorized with a mix of allowed and forbidden streams' do
+        @auth_headers = viewer.create_new_auth_token
+        get "/db_streams.json",
+          params: {streams: [stream1.id, stream2.id, other_stream.id].to_json},
+          headers: @auth_headers
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+    context 'without permissions' do
+      it 'returns unauthorized' do
+        @auth_headers = viewer.create_new_auth_token
+        get "/db_streams.json",
+          params: {streams: [other_stream.id].to_json},
+          headers: @auth_headers
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+    context 'without sign-in' do
+      it 'returns unauthorized' do
+        get "/db_streams.json",
+          params: {streams: [stream1.id, stream2.id].to_json}
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+  end
   # show action does not exist
   # show action does not exist
 
 
   describe 'PUT update' do
   describe 'PUT update' do

spec/factories/load_stream_data.rb

@@ -8,7 +8,7 @@ class MockLoadStreamData
     @data = nil
     @data = nil
     @run_count = 0
     @run_count = 0
   end
   end
-  def run(db_stream, start_time, end_time, elements=[])
+  def run(db_stream, start_time, end_time, elements=[], resolution=nil)
     @data = @dataset.select{|d| d[:stream]==db_stream}.first[:data]
     @data = @dataset.select{|d| d[:stream]==db_stream}.first[:data]
     @run_count += 1
     @run_count += 1
     if(@data == nil)
     if(@data == nil)

spec/services/data/load_stream_data_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe 'LoadStreamData' do
         expect(@service.success?).to be true
         expect(@service.success?).to be true
         expect(@service.data_type).to eq('decimated')
         expect(@service.data_type).to eq('decimated')
       end
       end
-      it 'finds appropriate level based on nilm resolution' do
+      it 'finds max allowed resolution by default' do
         # expect level 16 decimation to meet plotting requirements
         # expect level 16 decimation to meet plotting requirements
         @service.run(@db_stream, 10, 90)
         @service.run(@db_stream, 10, 90)
         expect(@mockAdapter.level_retrieved).to eq(16)
         expect(@mockAdapter.level_retrieved).to eq(16)
@@ -47,6 +47,19 @@ RSpec.describe 'LoadStreamData' do
         @service.run(@db_stream, 10, 90)
         @service.run(@db_stream, 10, 90)
         expect(@mockAdapter.level_retrieved).to eq(64)
         expect(@mockAdapter.level_retrieved).to eq(64)
       end
       end
+      it 'finds lower resolution if requested' do
+        # expect level 64 decimation to meet plotting requirements
+        @service.run(@db_stream, 10, 90, [], 50)
+        expect(@mockAdapter.level_retrieved).to eq(64)
+        # with higher resolution setting, level should stay the same
+        db.max_points_per_plot = 425; db.save
+        @service.run(@db_stream, 10, 90, [], 50)
+        expect(@mockAdapter.level_retrieved).to eq(64)
+        # when resolution > allowed, returns allowed
+        db.max_points_per_plot = 26; db.save
+        @service.run(@db_stream, 10, 90, [], 1000)
+        expect(@mockAdapter.level_retrieved).to eq(64)
+      end
       it 'populates @data structure with decimated data' do
       it 'populates @data structure with decimated data' do
         @service.run(@db_stream, 10, 90)
         @service.run(@db_stream, 10, 90)
         expect(@service.data.length).to eq 3
         expect(@service.data.length).to eq 3