Skip to content

wattsworth / lumen-api

Go to a project
Commit 0986afad by John Doe
Options

added nilm_id to json for db_streams

parent 4d2b1f7e
Inline Side-by-side
Showing with 152 additions and 33 deletions
app/adapters/db_adapter.rb
...@@ -3,7 +3,10 @@ ...@@ -3,7 +3,10 @@
# Wrapper around NilmDB HTTP service # Wrapper around NilmDB HTTP service
class DbAdapter class DbAdapter
include HTTParty include HTTParty
default_timeout 10 default_timeout 5
open_timeout 5
read_timeout 5
attr_reader :url attr_reader :url
def initialize(url) def initialize(url)
......
app/controllers/db_elements_controller.rb
...@@ -3,16 +3,7 @@ class DbElementsController < ApplicationController ...@@ -3,16 +3,7 @@ class DbElementsController < ApplicationController
before_action :authenticate_user! before_action :authenticate_user!
#def index
# @elements = DbElement.find(JSON.parse(params[:elements]))
# # make sure the user is allowed to view these elements
# @elements.each do |elem|
# unless current_user.views_nilm?(elem.db_stream.db.nilm)
# head :unauthorized
# return
# end
# end
#end
def data def data
req_elements = DbElement.find(JSON.parse(params[:elements])) req_elements = DbElement.find(JSON.parse(params[:elements]))
...@@ -26,12 +17,28 @@ class DbElementsController < ApplicationController ...@@ -26,12 +17,28 @@ class DbElementsController < ApplicationController
# make sure the time range makes sense # make sure the time range makes sense
start_time = (params[:start_time].to_i unless params[:start_time].nil?) start_time = (params[:start_time].to_i unless params[:start_time].nil?)
end_time = (params[:end_time].to_i unless params[:end_time].nil?) end_time = (params[:end_time].to_i unless params[:end_time].nil?)
#requested resolution (leave blank for max possible)
resolution = (params[:resolution].to_i unless params[:resolution].nil?)
# padding: percentage of data to retrieve beyond start|end
padding = params[:padding].nil? ? 0 : params[:padding].to_f
# retrieve the data for the requested elements # retrieve the data for the requested elements
@service = LoadElementData.new @service = LoadElementData.new
@service.run(req_elements, start_time, end_time)
#if start and end are specified, calculate padding
if !start_time.nil? && !end_time.nil?
actual_start = (start_time - (end_time-start_time)*padding).to_i
actual_end = (end_time + (end_time-start_time)*padding).to_i
@service.run(req_elements, actual_start, actual_end, resolution)
@start_time = start_time
@end_time = end_time
#otherwise let the service determine the start/end automatically
else
@service.run(req_elements, start_time, end_time, resolution)
@start_time = @service.start_time @start_time = @service.start_time
@end_time = @service.end_time @end_time = @service.end_time
end
render status: @service.success? ? :ok : :unprocessable_entity render status: @service.success? ? :ok : :unprocessable_entity
end end
......
app/controllers/db_streams_controller.rb
...@@ -7,6 +7,22 @@ class DbStreamsController < ApplicationController ...@@ -7,6 +7,22 @@ class DbStreamsController < ApplicationController
before_action :authorize_viewer, only: [:data] before_action :authorize_viewer, only: [:data]
before_action :authorize_owner, only: [:update] before_action :authorize_owner, only: [:update]
def index
if params[:streams].nil?
head :unprocessable_entity
return
end
@streams = DbStream.find(JSON.parse(params[:streams]))
# make sure the user is allowed to view these streams
@streams.each do |stream|
unless current_user.views_nilm?(stream.db.nilm)
head :unauthorized
return
end
end
end
def update def update
adapter = DbAdapter.new(@db.url) adapter = DbAdapter.new(@db.url)
@service = EditStream.new(adapter) @service = EditStream.new(adapter)
......
app/services/data/load_element_data.rb
...@@ -17,6 +17,8 @@ class LoadElementData ...@@ -17,6 +17,8 @@ class LoadElementData
# start_time and end_time are unix us # start_time and end_time are unix us
# if start_time is nil it is set to earliest timestamp # if start_time is nil it is set to earliest timestamp
# if end_time is nil it is set to latest timestamp # if end_time is nil it is set to latest timestamp
# if resolution is nil, retrieve highest resolution possible
# #
# sets data # sets data
# data: # data:
...@@ -24,7 +26,7 @@ class LoadElementData ...@@ -24,7 +26,7 @@ class LoadElementData
# {id: element_id, values: [...]},...] # {id: element_id, values: [...]},...]
# see load_stream_data for details on the value structure # see load_stream_data for details on the value structure
# #
def run(elements, start_time, end_time) def run(elements, start_time, end_time, resolution = nil)
#1 figure out what streams need to be pulled #1 figure out what streams need to be pulled
req_streams = [] req_streams = []
elements.each do |elem| elements.each do |elem|
...@@ -76,7 +78,7 @@ class LoadElementData ...@@ -76,7 +78,7 @@ class LoadElementData
adapter = DbAdapter.new(stream.db.url) adapter = DbAdapter.new(stream.db.url)
data_service = LoadStreamData.new(adapter) data_service = LoadStreamData.new(adapter)
stream_elements = elements.select{|e| e.db_stream_id==stream.id}.to_a stream_elements = elements.select{|e| e.db_stream_id==stream.id}.to_a
data_service.run(stream, @start_time, @end_time,stream_elements) data_service.run(stream, @start_time, @end_time,stream_elements,resolution)
if data_service.success? if data_service.success?
combined_data.concat(data_service.data) combined_data.concat(data_service.data)
......
app/services/data/load_stream_data.rb
...@@ -17,6 +17,8 @@ class LoadStreamData ...@@ -17,6 +17,8 @@ class LoadStreamData
# associated database, sets data and data_type # associated database, sets data and data_type
# specify a subset of elements as an optional array # specify a subset of elements as an optional array
# if ommitted, all elements are extracted from the stream (expensive!) # if ommitted, all elements are extracted from the stream (expensive!)
# optionally specify a resolution, if omitted, returns maximum resolution
# allowed by the nilm
# #
# sets data and data_type # sets data and data_type
# data_type: raw # data_type: raw
...@@ -33,7 +35,7 @@ class LoadStreamData ...@@ -33,7 +35,7 @@ class LoadStreamData
# data: # data:
# [{id: element_id, type: decimated, values: [[start,0],[end,0],nil,...]}] # [{id: element_id, type: decimated, values: [[start,0],[end,0],nil,...]}]
# #
def run(db_stream, start_time, end_time, elements = []) def run(db_stream, start_time, end_time, elements = [], resolution=nil)
# if elements are not explicitly passed, get all of them # if elements are not explicitly passed, get all of them
if(elements.empty?) if(elements.empty?)
...@@ -41,7 +43,11 @@ class LoadStreamData ...@@ -41,7 +43,11 @@ class LoadStreamData
end end
elements.sort_by!(&:column) elements.sort_by!(&:column)
resolution = db_stream.db.max_points_per_plot resolution = if resolution.nil?
db_stream.db.max_points_per_plot
else
[db_stream.db.max_points_per_plot,resolution].min
end
valid_decim = findValidDecimationLevel(db_stream, start_time) valid_decim = findValidDecimationLevel(db_stream, start_time)
# valid_decim is the highest resolution, find one we can plot # valid_decim is the highest resolution, find one we can plot
plottable_decim = findPlottableDecimationLevel( plottable_decim = findPlottableDecimationLevel(
...@@ -98,9 +104,8 @@ class LoadStreamData ...@@ -98,9 +104,8 @@ class LoadStreamData
# the decimation level will be 0 # the decimation level will be 0
# #
def findPlottableDecimationLevel( def findPlottableDecimationLevel(
db_stream, valid_decim, start_time, end_time, _resolution db_stream, valid_decim, start_time, end_time, resolution
) )
path = db_stream.path path = db_stream.path
path += "~decim-#{valid_decim.level}" if valid_decim.level > 1 path += "~decim-#{valid_decim.level}" if valid_decim.level > 1
# figure out how much data this stream has over the interval # figure out how much data this stream has over the interval
...@@ -112,16 +117,15 @@ class LoadStreamData ...@@ -112,16 +117,15 @@ class LoadStreamData
# find out how much raw data exists over the specified interval # find out how much raw data exists over the specified interval
raw_count = count * valid_decim.level raw_count = count * valid_decim.level
# now we can find the right decimation level for plotting # now we can find the right decimation level for plotting
max_count = db_stream.db.max_points_per_plot
# if the valid decim can be plotted, use it # if the valid decim can be plotted, use it
return valid_decim if raw_count <= max_count return valid_decim if raw_count <= resolution
# otherwise look for a higher decimation level # otherwise look for a higher decimation level
found_valid_decim = false found_valid_decim = false
db_stream.db_decimations db_stream.db_decimations
.where('level >= ?', valid_decim.level) .where('level >= ?', valid_decim.level)
.order(:level) .order(:level)
.each do |decim| .each do |decim|
if raw_count / decim.level <= max_count if raw_count / decim.level <= resolution
# the lowest decimation level is the best # the lowest decimation level is the best
return decim return decim
end end
......
app/views/db_folders/_db_folder.json.jbuilder
...@@ -6,6 +6,7 @@ end ...@@ -6,6 +6,7 @@ end
json.streams(db_folder.db_streams.includes(:db_elements)) do |stream| json.streams(db_folder.db_streams.includes(:db_elements)) do |stream|
json.extract! stream, *DbStream.json_keys json.extract! stream, *DbStream.json_keys
json.nilm_id nilm.id
json.elements(stream.db_elements) do |element| json.elements(stream.db_elements) do |element|
json.extract! element, *DbElement.json_keys json.extract! element, *DbElement.json_keys
end end
......
app/views/db_folders/show.json.jbuilder
# frozen_string_literal: true # frozen_string_literal: true
json.partial! 'db_folders/db_folder', json.partial! 'db_folders/db_folder',
db_folder: @db_folder db_folder: @db_folder, nilm: @nilm
app/views/db_streams/_db_stream.json.jbuilder
# frozen_string_literal: true # frozen_string_literal: true
json.extract! db_stream, *DbStream.json_keys json.extract! db_stream, *DbStream.json_keys
json.nilm_id db_stream.db.nilm.id
json.elements(db_stream.db_elements) do |element| json.elements(db_stream.db_elements) do |element|
json.extract! element, *DbElement.json_keys json.extract! element, *DbElement.json_keys
......
app/views/db_streams/data.csv.erb
############################################### ###############################################
# Stream: <%=@db_stream.name%> # Stream: <%=@db_stream.name%>
# Installation: <%=@nilm.name%> <%unless @nilm.description.blank?%>(<%=@nilm.description%>)<%end%>
# Path: <%=@db_stream.path%> # Path: <%=@db_stream.path%>
# Source: <%=@nilm.name%> <%unless @nilm.description.blank?%>(<%=@nilm.description%>)<%end%> # URL: <%=@nilm.url%>
#
# #
# start: <%=Time.at(@legend[:start_time]/1e6)%> # start: <%=Time.at(@legend[:start_time]/1e6)%>
# end: <%=Time.at(@legend[:end_time]/1e6)%> # end: <%=Time.at(@legend[:end_time]/1e6)%>
......
app/views/db_streams/index.json.jbuilder 0 → 100644
json.array! @streams do |stream|
json.extract! stream, *DbStream.json_keys
json.nilm_id stream.db.nilm.id
json.elements(stream.db_elements) do |element|
json.extract! element, *DbElement.json_keys
end
end
app/views/db_streams/show.json.jbuilder deleted 100644 → 0
# frozen_string_literal: true
json.partial! 'db_streams/db_stream',
db_folder: @db_stream
config/routes.rb
...@@ -12,7 +12,7 @@ Rails.application.routes.draw do ...@@ -12,7 +12,7 @@ Rails.application.routes.draw do
end end
resources :dbs, only: [:show, :update] resources :dbs, only: [:show, :update]
resources :db_folders, only: [:show, :update] resources :db_folders, only: [:show, :update]
resources :db_streams, only: [:update] do resources :db_streams, only: [:index, :update] do
member do member do
post 'data' post 'data'
end end
......
spec/controllers/db_elements_controller_spec.rb
...@@ -18,8 +18,6 @@ RSpec.describe DbElementsController, type: :request do ...@@ -18,8 +18,6 @@ RSpec.describe DbElementsController, type: :request do
stream.db_elements << @elem2 stream.db_elements << @elem2
end end
it "returns elements with data" do it "returns elements with data" do
@service_data = [{ id: @elem1.id, data: 'mock1' }, @service_data = [{ id: @elem1.id, data: 'mock1' },
{ id: @elem2.id, data: 'mock2' }] { id: @elem2.id, data: 'mock2' }]
@mock_service = instance_double(LoadElementData, @mock_service = instance_double(LoadElementData,
...@@ -40,6 +38,31 @@ RSpec.describe DbElementsController, type: :request do ...@@ -40,6 +38,31 @@ RSpec.describe DbElementsController, type: :request do
body = JSON.parse(response.body) body = JSON.parse(response.body)
expect(body['data'].count).to eq(2) expect(body['data'].count).to eq(2)
end end
it 'computes padding if specified' do
@service_data = [{ id: @elem1.id, data: 'mock1' },
{ id: @elem2.id, data: 'mock2' }]
@mock_service = instance_double(LoadElementData,
run: StubService.new,
success?: true, notices: [], warnings: [], errors: [],
data: @service_data)
allow(LoadElementData).to receive(:new).and_return(@mock_service)
expect(@mock_service).to receive(:run).with([@elem1,@elem2],90,210,nil)
@auth_headers = user1.create_new_auth_token
get '/db_elements/data.json',
params: { elements: [@elem1.id, @elem2.id].to_json,
start_time: 100, end_time: 200, padding: 0.1 },
headers: @auth_headers
expect(response).to have_http_status(:ok)
# check to make sure JSON renders the elements
body = JSON.parse(response.body)
expect(body['data'].count).to eq(2)
# reported time bounds should *NOT* include padding
body['data'].map do |data|
expect(data['start_time']).to eq 100
expect(data['end_time']).to eq 200
end
end
it 'returns error if time bounds are invalid' do it 'returns error if time bounds are invalid' do
@auth_headers = user1.create_new_auth_token @auth_headers = user1.create_new_auth_token
get '/db_elements/data.json', get '/db_elements/data.json',
......
spec/controllers/db_streams_controller_spec.rb
...@@ -12,7 +12,52 @@ RSpec.describe DbStreamsController, type: :request do ...@@ -12,7 +12,52 @@ RSpec.describe DbStreamsController, type: :request do
db: john_nilm.db) db: john_nilm.db)
end end
# index action does not exist describe 'GET index' do
let(:viewer) {create(:user)}
let(:nilm) {create(:nilm, viewers: [viewer])}
let(:db) {create(:db, nilm: nilm)}
let(:stream2) {create(:db_stream, db: db)}
let(:stream1) {create(:db_stream, db: db)}
let(:other_nilm) {create(:nilm)}
let(:other_db) {create(:db, nilm: other_nilm)}
let(:other_stream) {create(:db_stream, db: other_db)}
context 'with viewer permissions' do
it 'returns array of requested streams' do
@auth_headers = viewer.create_new_auth_token
get "/db_streams.json",
params: {streams: [stream1.id, stream2.id].to_json},
headers: @auth_headers
expect(response).to have_http_status(:ok)
# check to make sure JSON renders the streams
streams = JSON.parse(response.body)
expect(streams.count).to eq(2)
end
it 'returns unauthorized with a mix of allowed and forbidden streams' do
@auth_headers = viewer.create_new_auth_token
get "/db_streams.json",
params: {streams: [stream1.id, stream2.id, other_stream.id].to_json},
headers: @auth_headers
expect(response).to have_http_status(:unauthorized)
end
end
context 'without permissions' do
it 'returns unauthorized' do
@auth_headers = viewer.create_new_auth_token
get "/db_streams.json",
params: {streams: [other_stream.id].to_json},
headers: @auth_headers
expect(response).to have_http_status(:unauthorized)
end
end
context 'without sign-in' do
it 'returns unauthorized' do
get "/db_streams.json",
params: {streams: [stream1.id, stream2.id].to_json}
expect(response).to have_http_status(:unauthorized)
end
end
end
# show action does not exist # show action does not exist
describe 'PUT update' do describe 'PUT update' do
......
spec/factories/load_stream_data.rb
...@@ -8,7 +8,7 @@ class MockLoadStreamData ...@@ -8,7 +8,7 @@ class MockLoadStreamData
@data = nil @data = nil
@run_count = 0 @run_count = 0
end end
def run(db_stream, start_time, end_time, elements=[]) def run(db_stream, start_time, end_time, elements=[], resolution=nil)
@data = @dataset.select{|d| d[:stream]==db_stream}.first[:data] @data = @dataset.select{|d| d[:stream]==db_stream}.first[:data]
@run_count += 1 @run_count += 1
if(@data == nil) if(@data == nil)
......
spec/services/data/load_stream_data_spec.rb
...@@ -34,7 +34,7 @@ RSpec.describe 'LoadStreamData' do ...@@ -34,7 +34,7 @@ RSpec.describe 'LoadStreamData' do
expect(@service.success?).to be true expect(@service.success?).to be true
expect(@service.data_type).to eq('decimated') expect(@service.data_type).to eq('decimated')
end end
it 'finds appropriate level based on nilm resolution' do it 'finds max allowed resolution by default' do
# expect level 16 decimation to meet plotting requirements # expect level 16 decimation to meet plotting requirements
@service.run(@db_stream, 10, 90) @service.run(@db_stream, 10, 90)
expect(@mockAdapter.level_retrieved).to eq(16) expect(@mockAdapter.level_retrieved).to eq(16)
...@@ -47,6 +47,19 @@ RSpec.describe 'LoadStreamData' do ...@@ -47,6 +47,19 @@ RSpec.describe 'LoadStreamData' do
@service.run(@db_stream, 10, 90) @service.run(@db_stream, 10, 90)
expect(@mockAdapter.level_retrieved).to eq(64) expect(@mockAdapter.level_retrieved).to eq(64)
end end
it 'finds lower resolution if requested' do
# expect level 64 decimation to meet plotting requirements
@service.run(@db_stream, 10, 90, [], 50)
expect(@mockAdapter.level_retrieved).to eq(64)
# with higher resolution setting, level should stay the same
db.max_points_per_plot = 425; db.save
@service.run(@db_stream, 10, 90, [], 50)
expect(@mockAdapter.level_retrieved).to eq(64)
# when resolution > allowed, returns allowed
db.max_points_per_plot = 26; db.save
@service.run(@db_stream, 10, 90, [], 1000)
expect(@mockAdapter.level_retrieved).to eq(64)
end
it 'populates @data structure with decimated data' do it 'populates @data structure with decimated data' do
@service.run(@db_stream, 10, 90) @service.run(@db_stream, 10, 90)
expect(@service.data.length).to eq 3 expect(@service.data.length).to eq 3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment