merged with work code copy

app/adapters/db_adapter.rb

@@ -8,15 +8,16 @@ class DbAdapter
     @url = url
     @url = url
   end
   end
 
 
-  def schema
+  def schema # rubocop:disable Metrics/MethodLength
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump.parsed_response.map do |entry|
     dump.parsed_response.map do |entry|
       { path:       entry[0],
       { path:       entry[0],
-        type:       entry[1],
+        attributes: {
+          data_type:  entry[1],
           start_time: entry[2] || 0,
           start_time: entry[2] || 0,
           end_time:   entry[3] || 0,
           end_time:   entry[3] || 0,
           total_rows: entry[4],
           total_rows: entry[4],
-        total_time: entry[5],
+          total_time: entry[5] },
         metadata: {} }
         metadata: {} }
     end
     end
   end
   end

app/services/db/update_db.rb

 # frozen_string_literal: true
 # frozen_string_literal: true
 
 
 # Handles construction of database objects
 # Handles construction of database objects
-class UpdateDb
+class UpdateDb # rubocop:disable Metrics/ClassLength
+  attr_accessor :warnings, :errors
+
   def initialize(db:)
   def initialize(db:)
     @db = db
     @db = db
+    @warnings = []
+    @errors = []
   end
   end
 
 
   def run(db_adapter:)
   def run(db_adapter:)
@@ -26,6 +30,10 @@ class UpdateDb
   # Adds :chunks to each schema element
   # Adds :chunks to each schema element
   # :chunks is an array of the entry's path elements
   # :chunks is an array of the entry's path elements
   # this makes it easier to traverse the database structure.
   # this makes it easier to traverse the database structure.
+  # The array is reversed so the chunks can be popped off in order
+  #   path: '/data/meter/prep-a'
+  #   chunks: ['prep-a','meter','data']
+  #
   def __create_entries(schema)
   def __create_entries(schema)
     schema.map do |entry|
     schema.map do |entry|
       entry[:chunks] = entry[:path][1..-1].split('/').reverse
       entry[:chunks] = entry[:path][1..-1].split('/').reverse
@@ -46,7 +54,7 @@ class UpdateDb
     groups = __group_entries(entries)
     groups = __group_entries(entries)
     # process the groups as subfolders or files
     # process the groups as subfolders or files
     __process_folder_contents(folder, groups)
     __process_folder_contents(folder, groups)
-
+    # return the updated folder
     folder
     folder
   end
   end
 
 
@@ -54,6 +62,8 @@ class UpdateDb
   # use its metadata to update the folder's attributes
   # use its metadata to update the folder's attributes
   def  __read_info_entry(entries)
   def  __read_info_entry(entries)
     if entries[0][:chunks] == ['info']
     if entries[0][:chunks] == ['info']
+      # if there is an info entry, remove it from the array
+      # so we don't process it as a seperate file
       info_entry = entries.slice!(0)
       info_entry = entries.slice!(0)
       info_entry[:metadata]
       info_entry[:metadata]
     end
     end
@@ -89,14 +99,18 @@ class UpdateDb
   def __group_entries(entries)
   def __group_entries(entries)
     entry_groups = {}
     entry_groups = {}
     entries.map do |entry|
     entries.map do |entry|
-      # remove the ~decimXX ending so decimations are grouped
-      # with their base stream
-      group_name = entry[:chunks].pop.gsub(/~decim[\d]+$/, '')
+      # group streams by their base paths (ignore ~decim endings)
+      group_name = entry[:chunks].pop.gsub(decimation_tag, '')
       __add_to_group(entry_groups, group_name, entry)
       __add_to_group(entry_groups, group_name, entry)
     end
     end
     entry_groups
     entry_groups
   end
   end
 
 
+  # regex matching the ~decimXX ending on a stream path
+  def decimation_tag
+    /~decim([\d]+)$/
+  end
+
   # helper function to __group_entries that handles
   # helper function to __group_entries that handles
   # sorting entries into the entry_groups Hash
   # sorting entries into the entry_groups Hash
   def __add_to_group(entry_groups, group_name, entry)
   def __add_to_group(entry_groups, group_name, entry)
@@ -126,7 +140,7 @@ class UpdateDb
     # if the path's are the same up to a ~decimXX suffix
     # if the path's are the same up to a ~decimXX suffix
     # this is a file, otherwise return false
     # this is a file, otherwise return false
     num_files = entry_group.map { |entry|
     num_files = entry_group.map { |entry|
-      entry[:path].gsub(/~decim[\d]+$/, '')
+      entry[:path].gsub(decimation_tag, '')
     }.uniq.count
     }.uniq.count
     num_files == 1
     num_files == 1
   end
   end
@@ -136,6 +150,7 @@ class UpdateDb
   def __build_file(folder:, entry_group:,
   def __build_file(folder:, entry_group:,
                    default_name:)
                    default_name:)
     base = __base_entry(entry_group)
     base = __base_entry(entry_group)
+    return unless base # corrupt file, don't process
     # find or create the file
     # find or create the file
     file = folder.db_files.find_by_path(base[:path])
     file = folder.db_files.find_by_path(base[:path])
     file ||= DbFile.new(db_folder: folder, path: base[:path])
     file ||= DbFile.new(db_folder: folder, path: base[:path])
@@ -148,17 +163,23 @@ class UpdateDb
 
 
   # find the base stream in this entry_group
   # find the base stream in this entry_group
   # this is the stream that doesn't have a decimXX tag
   # this is the stream that doesn't have a decimXX tag
+  # adds a warning and returns nil if base entry is missing
   def __base_entry(entry_group)
   def __base_entry(entry_group)
-    entry_group.select { |entry|
-      entry[:path].match(/~decim[\d]+$/).nil?
+    base_entry = entry_group.select { |entry|
+      entry[:path].match(decimation_tag).nil?
     }.first
     }.first
+    unless base_entry
+      warnings << "Missing base stream for #{default_name} in #{folder.name}"
+      return nil
+    end
+    base_entry
   end
   end
 
 
   # create or update DbDecimations for the
   # create or update DbDecimations for the
   # specified DbFile
   # specified DbFile
   def __build_decimations(file:, entry_group:)
   def __build_decimations(file:, entry_group:)
     entry_group.each do |entry|
     entry_group.each do |entry|
-      level = entry[:path].match(/~decim([\d]+)$/)[1].to_i
+      level = entry[:path].match(decimation_tag)[1].to_i
       decim = file.db_decimations.find_by_level(level)
       decim = file.db_decimations.find_by_level(level)
       decim ||= DbDecimation.new(db_file: file, level: level)
       decim ||= DbDecimation.new(db_file: file, level: level)
       decim.update_attributes(entry[:metadata])
       decim.update_attributes(entry[:metadata])

db/migrate/20160524161457_add_data_type_to_db_files.rb

+class AddDataTypeToDbFiles < ActiveRecord::Migration
+  def change
+    add_column :db_files, :data_type, :string
+  end
+end

db/migrate/20160524161816_add_data_type_to_db_decimations.rb

+class AddDataTypeToDbDecimations < ActiveRecord::Migration
+  def change
+    add_column :db_decimations, :data_type, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 20160524023832) do
+ActiveRecord::Schema.define(version: 20160524161816) do
 
 
   create_table "db_decimations", force: :cascade do |t|
   create_table "db_decimations", force: :cascade do |t|
     t.integer  "start_time", limit: 8
     t.integer  "start_time", limit: 8
@@ -23,6 +23,7 @@ ActiveRecord::Schema.define(version: 20160524023832) do
     t.datetime "created_at",           null: false
     t.datetime "created_at",           null: false
     t.datetime "updated_at",           null: false
     t.datetime "updated_at",           null: false
     t.integer  "level"
     t.integer  "level"
+    t.string   "data_type"
   end
   end
 
 
   create_table "db_files", force: :cascade do |t|
   create_table "db_files", force: :cascade do |t|
@@ -36,6 +37,7 @@ ActiveRecord::Schema.define(version: 20160524023832) do
     t.integer  "end_time",     limit: 8
     t.integer  "end_time",     limit: 8
     t.integer  "total_rows",   limit: 8
     t.integer  "total_rows",   limit: 8
     t.integer  "total_time",   limit: 8
     t.integer  "total_time",   limit: 8
+    t.string   "data_type"
   end
   end
 
 
   create_table "db_folders", force: :cascade do |t|
   create_table "db_folders", force: :cascade do |t|

spec/adapters/db_adapter_spec.rb

@@ -6,9 +6,11 @@ describe DbAdapter do
   it 'retrieves basic schema', :vcr do
   it 'retrieves basic schema', :vcr do
     db = double(url: 'http://archive.wattsworth.net/nilmdb')
     db = double(url: 'http://archive.wattsworth.net/nilmdb')
     adapter = DbAdapter.new(db.url)
     adapter = DbAdapter.new(db.url)
-    adapter.schema.map do |entry|
-      expect(entry).to include(:path, :type, :start_time,
-                               :end_time, :total_rows, :total_time, :metadata)
+    adapter.schema.each do |entry|
+      expect(entry).to include(:path, :attributes, :metadata)
+      expect(entry[:attributes]).to(
+        include(:data_type, :start_time,
+                :end_time, :total_rows, :total_time))
     end
     end
   end
   end
 end
 end

spec/factories/db_schema_helper.rb

@@ -4,16 +4,22 @@
 # are usually returned by DbAdapter.schema
 # are usually returned by DbAdapter.schema
 class DbSchemaHelper
 class DbSchemaHelper
   # schema data
   # schema data
+  # rubocop:disable Metrics/MethodLength
   def entry(path, type: 'uint8_1', metadata: {}, stream_count: 0)
   def entry(path, type: 'uint8_1', metadata: {}, stream_count: 0)
     if stream_count > 0
     if stream_count > 0
       metadata[:db_streams_attributes] = __build_streams(stream_count)
       metadata[:db_streams_attributes] = __build_streams(stream_count)
     end
     end
 
 
-    { path: path, type: type,
-      start_time: 0, end_time: 0,
-      total_rows: 0, total_time: 0,
+    { path: path,
+      attributes: {
+        data_type: type,
+        start_time: 0,
+        end_time: 0,
+        total_rows: 0,
+        total_time: 0 },
       metadata: metadata }
       metadata: metadata }
   end
   end
+  # rubocop:enable Metrics/MethodLength
 
 
   # build stream hash for a file
   # build stream hash for a file
   def __build_streams(count)
   def __build_streams(count)