added support for event count limit to enable plots with large numbers of events

app/adapters/joule/adapter.rb

@@ -106,9 +106,9 @@ module Joule
     # === END ANNOTATIONS ===
     # === END ANNOTATIONS ===
 
 
     # === BEGIN EVENTS ===
     # === BEGIN EVENTS ===
-    def read_events(stream, start_time, end_time)
-      events = @backend.read_events(stream.joule_id, start_time, end_time)
-      {id: stream.id, valid: true, events: events}
+    def read_events(stream, max_events, start_time, end_time)
+      result = @backend.read_events(stream.joule_id, max_events, start_time, end_time)
+      {id: stream.id, valid: true, count: result[:count], events: result[:events]}
     end
     end
 
 
     def node_type
     def node_type

app/adapters/joule/backend.rb

@@ -261,10 +261,11 @@ module Joule
     end
     end
 
 
     # === EVENT METHODS ===
     # === EVENT METHODS ===
-    def read_events(stream_id, start_time, end_time)
+    def read_events(stream_id, max_events, start_time, end_time)
       query = {'id': stream_id}
       query = {'id': stream_id}
       query['start'] = start_time unless start_time.nil?
       query['start'] = start_time unless start_time.nil?
       query['end'] = end_time unless end_time.nil?
       query['end'] = end_time unless end_time.nil?
+      query['limit'] = max_events
       options = {query: query}
       options = {query: query}
       begin
       begin
         resp = self.class.get("#{@url}/event/data.json", options)
         resp = self.class.get("#{@url}/event/data.json", options)
@@ -272,7 +273,14 @@ module Joule
       rescue
       rescue
         raise "connection error"
         raise "connection error"
       end
       end
-      resp.parsed_response.map{|event| event.deep_symbolize_keys}
+      if resp.parsed_response.is_a?(Hash)
+        resp.parsed_response.deep_symbolize_keys!
+        resp
+      else # backwards compatibility
+        events = resp.parsed_response.map{|event| event.deep_symbolize_keys}
+        count = if events.nil? then 0 else events.length end
+        {count: count, events: events}
+      end
     end
     end
 
 
     def delete_annotation(annotation_id)
     def delete_annotation(annotation_id)

app/controllers/nilms_controller.rb

@@ -86,7 +86,7 @@ class NilmsController < ApplicationController
       params.permit(:name, :description, :url)
       params.permit(:name, :description, :url)
     end
     end
     def db_params
     def db_params
-      params.permit(:max_points_per_plot, :url)
+      params.permit(:max_points_per_plot, :max_events_per_plot, :url)
     end
     end
 
 
     #authorization based on nilms
     #authorization based on nilms

app/services/event_stream/read_events.rb

@@ -21,11 +21,11 @@ class ReadEvents
     @data = []
     @data = []
     event_streams.each do |stream|
     event_streams.each do |stream|
       adapter = NodeAdapterFactory.from_nilm(stream.db.nilm)
       adapter = NodeAdapterFactory.from_nilm(stream.db.nilm)
-      result = adapter.read_events(stream, @start_time, @end_time)
+      result = adapter.read_events(stream, stream.db.max_events_per_plot, @start_time, @end_time)
       if not result.nil?
       if not result.nil?
         @data.append(result)
         @data.append(result)
       else
       else
-        @data.append({id: stream.id, valid: false, events: nil})
+        @data.append({id: stream.id, valid: false, count: 0, events: nil})
         add_warning("unable to retrieve events for #{stream.path}")
         add_warning("unable to retrieve events for #{stream.path}")
       end
       end
     end
     end

app/views/events/data.json.jbuilder

@@ -2,6 +2,7 @@ json.data do
   json.array! @service.data.each do |event_stream|
   json.array! @service.data.each do |event_stream|
     json.id event_stream[:id]
     json.id event_stream[:id]
     json.valid event_stream[:valid]
     json.valid event_stream[:valid]
+    json.count event_stream[:count]
     json.events event_stream[:events]
     json.events event_stream[:events]
     json.start_time @start_time
     json.start_time @start_time
     json.end_time @end_time
     json.end_time @end_time

app/views/nilms/show.json.jbuilder

@@ -4,6 +4,7 @@ json.data do
   json.role @role
   json.role @role
   if @nilm.db != nil
   if @nilm.db != nil
     json.max_points_per_plot @nilm.db.max_points_per_plot
     json.max_points_per_plot @nilm.db.max_points_per_plot
+    json.max_events_per_plot @nilm.db.max_events_per_plot
     json.available @nilm.db.available
     json.available @nilm.db.available
     json.root_folder do
     json.root_folder do
       if @nilm.db.root_folder != nil
       if @nilm.db.root_folder != nil

db/migrate/20220104010931_add_max_events_per_plot_to_dbs.rb

+class AddMaxEventsPerPlotToDbs < ActiveRecord::Migration[6.0]
+  def change
+    add_column :dbs, :max_events_per_plot, :integer, default: 200
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 2021_03_03_014945) do
+ActiveRecord::Schema.define(version: 2022_01_04_010931) do
 
 
   create_table "data_apps", force: :cascade do |t|
   create_table "data_apps", force: :cascade do |t|
     t.string "name"
     t.string "name"
@@ -117,6 +117,7 @@ ActiveRecord::Schema.define(version: 2021_03_03_014945) do
     t.string "version"
     t.string "version"
     t.integer "max_points_per_plot", default: 3600
     t.integer "max_points_per_plot", default: 3600
     t.boolean "available"
     t.boolean "available"
+    t.integer "max_events_per_plot", default: 200
   end
   end
 
 
   create_table "event_streams", force: :cascade do |t|
   create_table "event_streams", force: :cascade do |t|

spec/adapters/joule/adapter_spec.rb

@@ -11,20 +11,20 @@ describe Joule::Adapter do
 
 
     raw = File.read(File.dirname(__FILE__) + "/events.json")
     raw = File.read(File.dirname(__FILE__) + "/events.json")
     json = JSON.parse(raw)
     json = JSON.parse(raw)
+    json.deep_symbolize_keys!
     expect(mock_backend).to receive(:read_events) { json }
     expect(mock_backend).to receive(:read_events) { json }
 
 
     nilm = FactoryBot.create(:nilm, name: "test")
     nilm = FactoryBot.create(:nilm, name: "test")
     stream = FactoryBot.create(:event_stream, db: nilm.db, db_folder: nilm.db.root_folder,
     stream = FactoryBot.create(:event_stream, db: nilm.db, db_folder: nilm.db.root_folder,
                                name: 'test_stream')
                                name: 'test_stream')
-    result = adapter.read_events(stream,1611421200000000, 1611421230000000)
+    result = adapter.read_events(stream, 200, 1611421200000000, 1611421230000000)
     expect(result[:id]).to eq stream.id
     expect(result[:id]).to eq stream.id
     expect(result[:valid]).to be true
     expect(result[:valid]).to be true
-
     expect(result[:events].length).to eq 4
     expect(result[:events].length).to eq 4
     expected_event = {
     expected_event = {
-        "start_time" => 1611421200000000,
-        "end_time" => 1611421205000000,
-        "content" => {"name" => "test event 0"}}
+        start_time: 1611421200000000,
+        end_time: 1611421205000000,
+        content: {name: "test event 0"}}
     expect(result[:events][0]).to eq expected_event
     expect(result[:events][0]).to eq expected_event
   end
   end
 
 

spec/adapters/joule/backend_spec.rb

@@ -55,15 +55,16 @@ describe Joule::Backend do
         backend = Joule::Backend.new(url, key)
         backend = Joule::Backend.new(url, key)
         start_time = 1611421210000000
         start_time = 1611421210000000
         end_time = 1611421235000000
         end_time = 1611421235000000
-        events = backend.read_events(2, start_time, end_time)
+        result = backend.read_events(2, 200, start_time, end_time)
         # should have 3 events
         # should have 3 events
-        expect(events.length).to eq 3
+        expect(result[:count]).to eq 3
+        expect(result[:events].length).to eq 3
         expected_event = {
         expected_event = {
             "start_time":1611421210000000,
             "start_time":1611421210000000,
             "end_time":1611421215000000,
             "end_time":1611421215000000,
             "content":{
             "content":{
                 "name":"test event 1"}}
                 "name":"test event 1"}}
-        expect(events[0]).to eq expected_event
+        expect(result[:events][0]).to eq expected_event
       end
       end
       it 'handles errors', :vcr do
       it 'handles errors', :vcr do
         backend = Joule::Backend.new(url, key)
         backend = Joule::Backend.new(url, key)

spec/adapters/joule/events.json

-[
-  {
-    "start_time":1611421200000000,
-    "end_time":1611421205000000,
-    "content":{
-      "name":"test event 0"
+{
+  "count": 4,
+  "events": [
+    {
+      "start_time": 1611421200000000,
+      "end_time": 1611421205000000,
+      "content": {
+        "name": "test event 0"
+      }
+    },
+    {
+      "start_time": 1611421210000000,
+      "end_time": 1611421215000000,
+      "content": {
+        "name": "test event 1"
+      }
+    },
+    {
+      "start_time": 1611421220000000,
+      "end_time": 1611421225000000,
+      "content": {
+        "name": "test event 2"
+      }
+    },
+    {
+      "start_time": 1611421230000000,
+      "end_time": 1611421235000000,
+      "content": {
+        "name": "test event 3"
+      }
     }
     }
-  },
-  {
-    "start_time":1611421210000000,
+  ]
+}
\ No newline at end of file
-    "end_time":1611421215000000,
-    "content":{
-      "name":"test event 1"
-    }
-  },
-  {
-    "start_time":1611421220000000,
-    "end_time":1611421225000000,
-    "content":{
-      "name":"test event 2"
-    }
-  },
-  {
-    "start_time":1611421230000000,
-    "end_time":1611421235000000,
-    "content":{
-      "name":"test event 3"
-    }
-  }
-]
\ No newline at end of file

spec/cassettes/Joule_Backend/Events/read_events/loads_events.yml

@@ -2,7 +2,7 @@
 http_interactions:
 http_interactions:
 - request:
 - request:
     method: get
     method: get
-    uri: https://127.0.0.1:3030/event/data.json?end=1611421235000000&id=2&start=1611421210000000
+    uri: https://127.0.0.1:3030/event/data.json?end=1611421235000000&id=2&limit=200&start=1611421210000000
     body:
     body:
       encoding: US-ASCII
       encoding: US-ASCII
       string: ''
       string: ''

spec/cassettes/ReadEvents/when_boundary_times_are_not_specified/updates_the_streams.yml

@@ -289,7 +289,7 @@ http_interactions:
   recorded_at: Mon, 08 Mar 2021 21:26:43 GMT
   recorded_at: Mon, 08 Mar 2021 21:26:43 GMT
 - request:
 - request:
     method: get
     method: get
-    uri: https://localhost:3030/event/data.json?id=4
+    uri: https://localhost:3030/event/data.json?id=4&limit=200
     body:
     body:
       encoding: US-ASCII
       encoding: US-ASCII
       string: ''
       string: ''
@@ -348,7 +348,7 @@ http_interactions:
   recorded_at: Mon, 08 Mar 2021 21:26:52 GMT
   recorded_at: Mon, 08 Mar 2021 21:26:52 GMT
 - request:
 - request:
     method: get
     method: get
-    uri: https://localhost:3030/event/data.json?id=1
+    uri: https://localhost:3030/event/data.json?id=1&limit=200
     body:
     body:
       encoding: US-ASCII
       encoding: US-ASCII
       string: ''
       string: ''

spec/models/event_stream_spec.rb

-require 'rails_helper'
-
-RSpec.describe EventStream, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
-end

spec/services/events/read_events_spec.rb

@@ -48,7 +48,7 @@ RSpec.describe 'ReadEvents' do
       expect(service.warnings.length).to eq 1
       expect(service.warnings.length).to eq 1
       expect(service.data).to eq [
       expect(service.data).to eq [
                                      @event_stream1_data,
                                      @event_stream1_data,
-                                     {id: @event_stream2.id, valid: false, events: nil}
+                                     {id: @event_stream2.id, valid: false, events: nil, count: 0}
                                  ]
                                  ]
       expect(@mock_adapter.event_run_count).to eq 2
       expect(@mock_adapter.event_run_count).to eq 2
     end
     end

spec/support/mock_adapter.rb

@@ -16,7 +16,7 @@ class MockAdapter
     end
     end
     {data: data, decimation_factor: 1}
     {data: data, decimation_factor: 1}
   end
   end
-  def read_events(event_stream,start_time, end_time)
+  def read_events(event_stream,max_events, start_time, end_time)
     data = @events.select{|d| d[:event_stream]==event_stream}.first[:data]
     data = @events.select{|d| d[:event_stream]==event_stream}.first[:data]
     @event_run_count += 1
     @event_run_count += 1
     data
     data