added time and data size to joule backend and fixed data retrieval bugs

app/adapters/joule/backend.rb

@@ -82,10 +82,10 @@ module Joule
     end
     end
 
 
     def load_data(joule_id, start_time, end_time, resolution)
     def load_data(joule_id, start_time, end_time, resolution)
-      options = { query: { "id": joule_id,
-                           "start": start_time,
-                           "end": end_time,
-                           "max-rows": resolution}}
+      query = {'id': joule_id, 'max-rows': resolution}
+      query['start'] = start_time unless start_time.nil?
+      query['end'] = end_time unless end_time.nil?
+      options = { query: query}
       begin
       begin
         resp = self.class.get("#{@url}/data.json", options)
         resp = self.class.get("#{@url}/data.json", options)
         #TODO: handle interval data
         #TODO: handle interval data

app/adapters/joule/load_stream_data.rb

@@ -27,6 +27,10 @@ module Joule
                      [db_stream.db.max_points_per_plot,resolution].min
                      [db_stream.db.max_points_per_plot,resolution].min
                    end
                    end
       result = @backend.load_data(db_stream.joule_id, start_time, end_time, resolution)
       result = @backend.load_data(db_stream.joule_id, start_time, end_time, resolution)
+      if result.nil?
+        add_error("cannot get data for [#{db_stream.name}] @ #{@db_backend.url}")
+        return self
+      end
       # convert data into single array with nil's at interval boundaries
       # convert data into single array with nil's at interval boundaries
       data = []
       data = []
       result[:data].each do |interval|
       result[:data].each do |interval|

app/adapters/joule/update_db.rb

@@ -39,10 +39,28 @@ module Joule
       #puts db_folder.parent.id
       #puts db_folder.parent.id
       # update or create subfolders
       # update or create subfolders
       updated_ids = []
       updated_ids = []
+      size_on_disk = 0
+      start_time = nil
+      end_time = nil
       schema[:children].each do |child_schema|
       schema[:children].each do |child_schema|
         child = db_folder.subfolders.find_by_joule_id(child_schema[:id])
         child = db_folder.subfolders.find_by_joule_id(child_schema[:id])
         child ||= DbFolder.new(parent: db_folder, db: db_folder.db)
         child ||= DbFolder.new(parent: db_folder, db: db_folder.db)
         __update_folder(child, child_schema, db_folder.path)
         __update_folder(child, child_schema, db_folder.path)
+        size_on_disk+=child.size_on_disk
+        unless child.start_time.nil?
+          if start_time.nil?
+            start_time = child.start_time
+          else
+            start_time = [child.start_time, start_time].min
+          end
+        end
+        unless child.end_time.nil?
+          if end_time.nil?
+            end_time = child.end_time
+          else
+            end_time = [child.end_time, end_time].max
+          end
+        end
         updated_ids << child_schema[:id]
         updated_ids << child_schema[:id]
       end
       end
       # remove any subfolders that are no longer on the folder
       # remove any subfolders that are no longer on the folder
@@ -54,10 +72,30 @@ module Joule
         stream = db_folder.db_streams.find_by_joule_id(stream_schema[:id])
         stream = db_folder.db_streams.find_by_joule_id(stream_schema[:id])
         stream ||= DbStream.new(db_folder: db_folder, db: db_folder.db)
         stream ||= DbStream.new(db_folder: db_folder, db: db_folder.db)
         __update_stream(stream, stream_schema, db_folder.path)
         __update_stream(stream, stream_schema, db_folder.path)
+        size_on_disk+=stream.size_on_disk
+        unless stream.start_time.nil?
+          if start_time.nil?
+            start_time = stream.start_time
+          else
+            start_time = [stream.start_time, start_time].min
+          end
+        end
+        unless stream.end_time.nil?
+          if end_time.nil?
+            end_time = stream.end_time
+          else
+            end_time = [stream.end_time, end_time].max
+          end
+        end
         updated_ids << stream_schema[:id]
         updated_ids << stream_schema[:id]
       end
       end
       # remove any streams that are no longer in the folder
       # remove any streams that are no longer in the folder
       db_folder.db_streams.where.not(joule_id: updated_ids).destroy_all
       db_folder.db_streams.where.not(joule_id: updated_ids).destroy_all
+      # save the new disk size
+      db_folder.size_on_disk = size_on_disk
+      db_folder.start_time = start_time
+      db_folder.end_time = end_time
+      db_folder.save
     end
     end
 
 
     def __update_stream(db_stream, schema, parent_path)
     def __update_stream(db_stream, schema, parent_path)
@@ -66,16 +104,23 @@ module Joule
       attrs[:path] = "#{parent_path}/#{schema[:name]}"
       attrs[:path] = "#{parent_path}/#{schema[:name]}"
       attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}"
       attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}"
       attrs[:joule_id] = schema[:id]
       attrs[:joule_id] = schema[:id]
-      attrs[:total_time] = 100 # non-zero TODO, fix load_element so we don't need this
+      attrs[:start_time] = schema[:data_info][:start]
+      attrs[:end_time] = schema[:data_info][:end]
+      attrs[:total_rows] = schema[:data_info][:rows]
+      attrs[:total_time] = schema[:data_info][:total_time]
+      attrs[:size_on_disk] = schema[:data_info][:bytes]
+
       db_stream.update_attributes(attrs)
       db_stream.update_attributes(attrs)
-      db_stream.db_elements.destroy_all
+      #db_stream.db_elements.destroy_all
       schema[:elements].each do |element_config|
       schema[:elements].each do |element_config|
+        element = db_stream.db_elements.find_by_column(element_config[:index])
+        element ||= DbElement.new(db_stream: db_stream)
         attrs = element_config.slice(*DbElement.defined_attributes)
         attrs = element_config.slice(*DbElement.defined_attributes)
         # add in extra attributes that require conversion
         # add in extra attributes that require conversion
         attrs[:display_type] = element_config[:display_type].downcase
         attrs[:display_type] = element_config[:display_type].downcase
         attrs[:column] = element_config[:index]
         attrs[:column] = element_config[:index]
         attrs[:plottable] = true
         attrs[:plottable] = true
-        db_stream.db_elements << DbElement.new(attrs)
+        element.update_attributes(attrs)
       end
       end
     end
     end
 
 

app/adapters/nilmdb/update_stream.rb

@@ -34,8 +34,8 @@ module Nilmdb
       unless stream.update_attributes(base_entry[:attributes])
       unless stream.update_attributes(base_entry[:attributes])
         stream.use_default_attributes
         stream.use_default_attributes
         Rails.logger.warn("corrupt metadata: #{stream.path}")
         Rails.logger.warn("corrupt metadata: #{stream.path}")
-
       end
       end
+
       __compute_extents([base_entry] + decimation_entries)
       __compute_extents([base_entry] + decimation_entries)
       stream.start_time = @start_time
       stream.start_time = @start_time
       stream.end_time = @end_time
       stream.end_time = @end_time

app/services/data/load_element_data.rb

@@ -66,7 +66,7 @@ class LoadElementData
     #     .first.end_time
     #     .first.end_time
     # end
     # end
     # ------------------------- END MODIFICATION ------------------------------
     # ------------------------- END MODIFICATION ------------------------------
-    
+
     @start_time = start_time
     @start_time = start_time
     @end_time = end_time
     @end_time = end_time
     if (not @start_time.nil?) and (not @end_time.nil?) and (@start_time > @end_time)
     if (not @start_time.nil?) and (not @end_time.nil?) and (@start_time > @end_time)