can handle decimations

app/adapters/db_adapter.rb

@@ -8,15 +8,16 @@ class DbAdapter
     @url = url
     @url = url
   end
   end
 
 
-  def schema
+  def schema # rubocop:disable Metrics/MethodLength
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump.parsed_response.map do |entry|
     dump.parsed_response.map do |entry|
       { path:       entry[0],
       { path:       entry[0],
-        type:       entry[1],
-        start_time: entry[2] || 0,
-        end_time:   entry[3] || 0,
-        total_rows: entry[4],
-        total_time: entry[5],
+        attributes: {
+          data_type:  entry[1],
+          start_time: entry[2] || 0,
+          end_time:   entry[3] || 0,
+          total_rows: entry[4],
+          total_time: entry[5] },
         metadata: {} }
         metadata: {} }
     end
     end
   end
   end

app/services/db/update_db.rb

@@ -2,8 +2,12 @@
 
 
 # Handles construction of database objects
 # Handles construction of database objects
 class UpdateDb
 class UpdateDb
+  attr_accessor :warnings, :errors
+
   def initialize(db:)
   def initialize(db:)
     @db = db
     @db = db
+    @warnings = []
+    @errors = []
   end
   end
 
 
   def run(db_adapter:)
   def run(db_adapter:)
@@ -89,14 +93,18 @@ class UpdateDb
   def __group_entries(entries)
   def __group_entries(entries)
     entry_groups = {}
     entry_groups = {}
     entries.map do |entry|
     entries.map do |entry|
-      # remove the ~decimXX ending so decimations are grouped
-      # with their base stream
-      group_name = entry[:chunks].pop.gsub(/~decim[\d]+$/, '')
+      # group streams by their base paths
+      group_name = entry[:chunks].pop.gsub(decimation_tag, '')
       __add_to_group(entry_groups, group_name, entry)
       __add_to_group(entry_groups, group_name, entry)
     end
     end
     entry_groups
     entry_groups
   end
   end
 
 
+  # regex matching the ~decimXX ending on a stream path
+  def decimation_tag
+    /~decim([\d]+)$/
+  end
+
   # helper function to __group_entries that handles
   # helper function to __group_entries that handles
   # sorting entries into the entry_groups Hash
   # sorting entries into the entry_groups Hash
   def __add_to_group(entry_groups, group_name, entry)
   def __add_to_group(entry_groups, group_name, entry)
@@ -111,11 +119,14 @@ class UpdateDb
   # convert the groups into subfolders and files
   # convert the groups into subfolders and files
   def __process_folder_contents(folder, groups)
   def __process_folder_contents(folder, groups)
     groups.each do |name, entry_group|
     groups.each do |name, entry_group|
-      # TODO
       # if all paths in the entry group are the same up to a ~decim
       # if all paths in the entry group are the same up to a ~decim
-      # then this is a file, otherwise this is a subfolder
-      if entry_group.length == 1
-        __build_file(folder: folder, entry: entry_group[0], default_name: name)
+      # then this is a file
+      base_paths = entry_group.map do |entry|
+        entry[:path].gsub(decimation_tag, '')
+      end
+      if base_paths.uniq.count == 1
+        __build_file(folder: folder, entries: entry_group, default_name: name)
+      # otherwise this is a subfolder
       elsif entry_group.length > 1
       elsif entry_group.length > 1
         __parse_folder_entries(parent: folder, entries: entry_group,
         __parse_folder_entries(parent: folder, entries: entry_group,
                                default_name: name)
                                default_name: name)
@@ -125,12 +136,36 @@ class UpdateDb
 
 
   # create or update a DbFile object at the
   # create or update a DbFile object at the
   # specified path.
   # specified path.
-  def __build_file(folder:, entry:, default_name:)
-    file = folder.db_files.find_by_path(entry[:path])
-    file ||= DbFile.new(db_folder: folder, path: entry[:path])
-    info = { name: default_name }.merge(entry[:metadata])
-    file.update_attributes(info)
+  def __build_file(folder:, entries:, default_name:)
+    # find the base file entry
+    base_entry = entries.select { |entry| !entry[:path].match(decimation_tag) }
+    unless base_entry.count == 1
+      warnings << "Missing base stream for #{default_name} in #{folder.name}"
+      return
+    end
+    base_entry = base_entry.first
+    # find or create the DbFile object
+    file = folder.db_files.find_by_path(base_entry[:path])
+    file ||= DbFile.new(db_folder: folder, path: base_entry[:path])
+    # update the file info
+    info = { name: default_name }.merge(base_entry[:metadata])
+    file.update_attributes(info.merge(base_entry[:attributes]))
     file.save!
     file.save!
-    file
+    # add the decimations
+    decim_entries = entries.select do |entry|
+      entry[:path].match(decimation_tag)
+    end
+    __build_decimations(file: file, entries: decim_entries)
+  end
+
+  # create or update DbDecimation objects for a DbFile
+  def __build_decimations(file:, entries:)
+    entries.each do |entry|
+      level = entry[:path].match(decimation_tag)[1].to_i
+      decim = file.db_decimations.find_by_level(level)
+      decim ||= DbDecimation.new(db_file: file, level: level)
+      decim.update_attributes(entry[:attributes])
+      decim.save!
+    end
   end
   end
 end
 end

db/migrate/20160524161457_add_data_type_to_db_files.rb

+class AddDataTypeToDbFiles < ActiveRecord::Migration
+  def change
+    add_column :db_files, :data_type, :string
+  end
+end

db/migrate/20160524161816_add_data_type_to_db_decimations.rb

+class AddDataTypeToDbDecimations < ActiveRecord::Migration
+  def change
+    add_column :db_decimations, :data_type, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 20160524023832) do
+ActiveRecord::Schema.define(version: 20160524161816) do
 
 
   create_table "db_decimations", force: :cascade do |t|
   create_table "db_decimations", force: :cascade do |t|
     t.integer  "start_time", limit: 8
     t.integer  "start_time", limit: 8
@@ -23,6 +23,7 @@ ActiveRecord::Schema.define(version: 20160524023832) do
     t.datetime "created_at",           null: false
     t.datetime "created_at",           null: false
     t.datetime "updated_at",           null: false
     t.datetime "updated_at",           null: false
     t.integer  "level"
     t.integer  "level"
+    t.string   "data_type"
   end
   end
 
 
   create_table "db_files", force: :cascade do |t|
   create_table "db_files", force: :cascade do |t|
@@ -36,6 +37,7 @@ ActiveRecord::Schema.define(version: 20160524023832) do
     t.integer  "end_time",     limit: 8
     t.integer  "end_time",     limit: 8
     t.integer  "total_rows",   limit: 8
     t.integer  "total_rows",   limit: 8
     t.integer  "total_time",   limit: 8
     t.integer  "total_time",   limit: 8
+    t.string   "data_type"
   end
   end
 
 
   create_table "db_folders", force: :cascade do |t|
   create_table "db_folders", force: :cascade do |t|

spec/adapters/db_adapter_spec.rb

@@ -6,9 +6,11 @@ describe DbAdapter do
   it 'retrieves basic schema', :vcr do
   it 'retrieves basic schema', :vcr do
     db = double(url: 'http://archive.wattsworth.net/nilmdb')
     db = double(url: 'http://archive.wattsworth.net/nilmdb')
     adapter = DbAdapter.new(db.url)
     adapter = DbAdapter.new(db.url)
-    adapter.schema.map do |entry|
-      expect(entry).to include(:path, :type, :start_time,
-                               :end_time, :total_rows, :total_time, :metadata)
+    adapter.schema.each do |entry|
+      expect(entry).to include(:path, :attributes, :metadata)
+      expect(entry[:attributes]).to(
+        include(:data_type, :start_time,
+                :end_time, :total_rows, :total_time))
     end
     end
   end
   end
 end
 end

spec/factories/db_schema_helper.rb

@@ -4,16 +4,22 @@
 # are usually returned by DbAdapter.schema
 # are usually returned by DbAdapter.schema
 class DbSchemaHelper
 class DbSchemaHelper
   # schema data
   # schema data
+  # rubocop:disable Metrics/MethodLength
   def entry(path, type: 'uint8_1', metadata: {}, stream_count: 0)
   def entry(path, type: 'uint8_1', metadata: {}, stream_count: 0)
     if stream_count > 0
     if stream_count > 0
       metadata[:db_streams_attributes] = __build_streams(stream_count)
       metadata[:db_streams_attributes] = __build_streams(stream_count)
     end
     end
 
 
-    { path: path, type: type,
-      start_time: 0, end_time: 0,
-      total_rows: 0, total_time: 0,
+    { path: path,
+      attributes: {
+        data_type: type,
+        start_time: 0,
+        end_time: 0,
+        total_rows: 0,
+        total_time: 0 },
       metadata: metadata }
       metadata: metadata }
   end
   end
+  # rubocop:enable Metrics/MethodLength
 
 
   # build stream hash for a file
   # build stream hash for a file
   def __build_streams(count)
   def __build_streams(count)