working on database parsing

app/adapters/db_adapter.rb

@@ -9,11 +9,23 @@ class DbAdapter
   end
 
   def schema # rubocop:disable Metrics/MethodLength
+    # GET extended info stream list
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump.parsed_response.map do |entry|
       # TODO: implement global metadata dump, because this is sloooow
+      # GET metadata for the stream
       dump = self.class.get("#{@url}/stream/get_metadata?path=#{entry[0]}")
-      metadata = dump.parsed_response['config_key__']
+      metadata = JSON.parse(dump.parsed_response['config_key__'] || '{}')
+      # The streams are not pure attributes, pull them out
+      streams = metadata["streams"] || {}
+      # Add plain-text metadata keys (retrofit for *info streams which keep
+      # attributes in seperate metadata tags
+      metadata.merge!(dump.parsed_response.slice("delete_locked",
+                                                "description",
+                                                "hidden",
+                                                "name"))
+      # Create the schema:
+      # 3 elements: path, attributes, streams
       { path:       entry[0],
         attributes: {
           data_type:  entry[1],
@@ -21,7 +33,8 @@ class DbAdapter
           end_time:   entry[3] || 0,
           total_rows: entry[4],
           total_time: entry[5]
-        }.merge(metadata)
+        }.merge(metadata.except("streams")),
+        streams: streams
       }
     end
   end

app/models/db_decimation.rb

@@ -3,4 +3,9 @@
 # Decimation level of a file
 class DbDecimation < ActiveRecord::Base
   belongs_to :db_file
+
+  def as_json(_options = {})
+    super(except: [:created_at, :updated_at])
+  end
+
 end

app/models/db_file.rb

@@ -5,7 +5,7 @@ class DbFile < ActiveRecord::Base
   belongs_to :db_folder
   has_many :db_streams, dependent: :destroy
   has_many :db_decimations, dependent: :destroy
-  accepts_nested_attributes_for :db_streams
+
 
   def remove(db_service:)
     db_service.remove_file(path)
@@ -13,6 +13,9 @@ class DbFile < ActiveRecord::Base
   end
 
   def as_json(_options = {})
-    super(except: [:created_at, :updated_at])
+    file = super(except: [:created_at, :updated_at])
+    file[:streams] = db_streams.map(&:as_json)
+    file[:decimations] = db_decimations.map(&:as_json)
+    file
   end
 end

app/models/db_stream.rb

@@ -4,4 +4,9 @@
 # in the db hierarchy and contains actual data
 class DbStream < ActiveRecord::Base
   belongs_to :db_file
+
+  def as_json(_options = {})
+    super(except: [:created_at, :updated_at])
+  end
+
 end

app/services/db/update_db.rb

@@ -65,7 +65,7 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
       # if there is an info entry, remove it from the array
       # so we don't process it as a seperate file
       info_entry = entries.slice!(0)
-      info_entry[:metadata]
+      info_entry[:attributes]
     end
   end
 
@@ -86,7 +86,7 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
     return @root_folder if parent.nil?
     folder = parent.subfolders.find_by_path(path)
     folder ||= DbFolder.new(parent: parent, path: path)
-    folder.update_attributes(info)
+    folder.update_attributes(info.slice(:name))
     folder.save!
     folder
   end
@@ -108,7 +108,7 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
 
   # regex matching the ~decimXX ending on a stream path
   def decimation_tag
-    /~decim([\d]+)$/
+    /~decim-([\d]+)$/
   end
 
   # helper function to __group_entries that handles
@@ -157,12 +157,15 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
     # find or create the file
     file = folder.db_files.find_by_path(base[:path])
     file ||= DbFile.new(db_folder: folder, path: base[:path])
-    info = { name: default_name }.merge(base[:metadata])
+    # if the file doesn't have a name, use the default
+    base[:attributes][:name] ||= default_name
     # automatically updates the streams for this file
-    file.update_attributes(info)
+    byebug if base[:attributes][:streams] != nil
+    file.update_attributes(base[:attributes])
     file.save!
     __build_decimations(file: file,
                         entry_group: entry_group - [base])
+    __build_streams(file: file, stream_data: base[:streams])
   end
 
   # find the base stream in this entry_group
@@ -183,8 +186,20 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
       level = entry[:path].match(decimation_tag)[1].to_i
       decim = file.db_decimations.find_by_level(level)
       decim ||= DbDecimation.new(db_file: file, level: level)
-      decim.update_attributes(entry[:metadata])
+      decim.update_attributes(entry[:attributes])
       decim.save!
     end
   end
+
+  # create or update DbStreams for the
+  # specified DbFile
+  def __build_streams(file:, stream_data:)
+    return if stream_data.empty?
+    stream_data.each do |entry|
+      stream = file.db_streams.find_by_column(entry[:column])
+      stream ||= DbStream.new(db_file: file)
+      stream.update_attributes(entry)
+      stream.save!
+    end
+  end
 end

db/migrate/20160709231829_add_name_abbrev_to_db_file.rb

+class AddNameAbbrevToDbFile < ActiveRecord::Migration
+  def change
+    add_column :db_files, :name_abbrev, :string
+  end
+end

db/migrate/20160709232003_add_delete_locked_to_db_file.rb

+class AddDeleteLockedToDbFile < ActiveRecord::Migration
+  def change
+    add_column :db_files, :delete_locked, :boolean
+  end
+end

db/migrate/20160709232731_rename_scale_to_scale_factor_in_db_streams.rb

+class RenameScaleToScaleFactorInDbStreams < ActiveRecord::Migration
+  def change
+    rename_column :db_streams, :scale, :scale_factor
+  end
+end

db/migrate/20160709232846_add_plottable_to_db_stream.rb

+class AddPlottableToDbStream < ActiveRecord::Migration
+  def change
+    add_column :db_streams, :plottable, :boolean
+    add_column :db_streams, :discrete, :boolean
+  end
+end

db/migrate/20160709234451_remove_int_from_db_decimations.rb

+class RemoveIntFromDbDecimations < ActiveRecord::Migration
+  def change
+    remove_column :db_decimations, :int
+  end
+end

db/migrate/20160709235828_add_hidden_to_db_file.rb

+class AddHiddenToDbFile < ActiveRecord::Migration
+  def change
+    add_column :db_files, :hidden, :boolean
+    add_column :db_folders, :hidden, :boolean
+  end
+end

db/schema.rb

@@ -11,11 +11,10 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160524161816) do
+ActiveRecord::Schema.define(version: 20160709235828) do
 
   create_table "db_decimations", force: :cascade do |t|
     t.integer  "start_time", limit: 8
-    t.integer  "int"
     t.integer  "end_time",   limit: 8
     t.integer  "total_rows", limit: 8
     t.integer  "total_time", limit: 8
@@ -30,14 +29,17 @@ ActiveRecord::Schema.define(version: 20160524161816) do
     t.string   "name"
     t.string   "description"
     t.integer  "db_folder_id"
-    t.datetime "created_at",             null: false
-    t.datetime "updated_at",             null: false
+    t.datetime "created_at",              null: false
+    t.datetime "updated_at",              null: false
     t.string   "path"
-    t.integer  "start_time",   limit: 8
-    t.integer  "end_time",     limit: 8
-    t.integer  "total_rows",   limit: 8
-    t.integer  "total_time",   limit: 8
+    t.integer  "start_time",    limit: 8
+    t.integer  "end_time",      limit: 8
+    t.integer  "total_rows",    limit: 8
+    t.integer  "total_time",    limit: 8
     t.string   "data_type"
+    t.string   "name_abbrev"
+    t.boolean  "delete_locked"
+    t.boolean  "hidden"
   end
 
   create_table "db_folders", force: :cascade do |t|
@@ -47,6 +49,7 @@ ActiveRecord::Schema.define(version: 20160524161816) do
     t.datetime "updated_at",  null: false
     t.integer  "parent_id"
     t.string   "path"
+    t.boolean  "hidden"
   end
 
   create_table "db_streams", force: :cascade do |t|
@@ -55,11 +58,13 @@ ActiveRecord::Schema.define(version: 20160524161816) do
     t.integer  "column"
     t.float    "default_max"
     t.float    "default_min"
-    t.float    "scale"
+    t.float    "scale_factor"
     t.float    "offset"
     t.integer  "db_file_id"
-    t.datetime "created_at",  null: false
-    t.datetime "updated_at",  null: false
+    t.datetime "created_at",   null: false
+    t.datetime "updated_at",   null: false
+    t.boolean  "plottable"
+    t.boolean  "discrete"
   end
 
   create_table "dbs", force: :cascade do |t|

spec/adapters/db_adapter_spec.rb

@@ -4,10 +4,11 @@ require 'rails_helper'
 
 describe DbAdapter do
   it 'retrieves basic schema', :vcr do
-    db = double(url: 'http://archive.wattsworth.net/nilmdb')
+    # use the vagrant box loaded with example database
+    db = double(url: 'http://localhost:8080/nilmdb')
     adapter = DbAdapter.new(db.url)
     adapter.schema.each do |entry|
-      expect(entry).to include(:path, :attributes, :metadata)
+      expect(entry).to include(:path, :attributes)
       expect(entry[:attributes]).to(
         include(:data_type, :start_time,
                 :end_time, :total_rows, :total_time))

spec/factories/db_schema_helper.rb

@@ -6,24 +6,23 @@ class DbSchemaHelper
   # schema data
   # rubocop:disable Metrics/MethodLength
   def entry(path, type: 'uint8_1', metadata: {}, stream_count: 0)
-    if stream_count > 0
-      metadata[:db_streams_attributes] = __build_streams(stream_count)
-    end
-
-    { path: path,
+    data = {
+      path: path,
       attributes: {
         data_type: type,
         start_time: 0,
         end_time: 0,
         total_rows: 0,
-        total_time: 0 },
-      metadata: metadata }
+        total_time: 0 }.merge(metadata),
+      streams: __build_streams(stream_count)
+    }
+
   end
   # rubocop:enable Metrics/MethodLength
 
   # build stream hash for a file
   def __build_streams(count)
-    return nil unless count > 0
+    return {} unless count > 0
     streams = []
     (0..(count - 1)).each do |i|
       streams <<

spec/models/db_file_spec.rb

@@ -5,8 +5,11 @@ RSpec.describe 'DbFile' do
   describe 'object' do
     let(:db_file) { DbFile.new }
     specify { expect(db_file).to respond_to(:name) }
+    specify { expect(db_file).to respond_to(:name_abbrev) }
     specify { expect(db_file).to respond_to(:description) }
     specify { expect(db_file).to respond_to(:db_streams) }
+    specify { expect(db_file).to respond_to(:hidden) }
+
   end
 
   describe 'remove' do

spec/models/db_folder_spec.rb

@@ -9,6 +9,8 @@ RSpec.describe 'DbFolder' do
     specify { expect(db_folder).to respond_to(:parent) }
     specify { expect(db_folder).to respond_to(:subfolders) }
     specify { expect(db_folder).to respond_to(:db_files) }
+    specify { expect(db_folder).to respond_to(:hidden) }
+
   end
 
   describe 'insert_file' do

spec/models/db_stream_spec.rb

@@ -9,7 +9,10 @@ RSpec.describe 'DbStream' do
     specify { expect(db_stream).to respond_to(:column) }
     specify { expect(db_stream).to respond_to(:default_max) }
     specify { expect(db_stream).to respond_to(:default_min) }
-    specify { expect(db_stream).to respond_to(:scale) }
+    specify { expect(db_stream).to respond_to(:scale_factor) }
     specify { expect(db_stream).to respond_to(:offset) }
+    specify { expect(db_stream).to respond_to(:plottable) }
+    specify { expect(db_stream).to respond_to(:discrete) }
+
   end
 end

spec/services/db/update_db_spec.rb

@@ -14,12 +14,12 @@ helper = DbSchemaHelper.new
 
 simple_db = [
   helper.entry('/folder1/f1_1',
-               metadata: { name: 'file1_1' }, stream_count: 3),
+               metadata: { name: 'file1_1' }, stream_count: 4),
   helper.entry('/folder1/f1_2',
                metadata: { name: 'file1_2' }, stream_count: 5),
   helper.entry('/folder2/f2_1',
                metadata: { name: 'file2_1' }, stream_count: 1),
-  helper.entry('/folder2/f2_2', 
+  helper.entry('/folder2/f2_2',
                metadata: { name: 'file2_2' }, stream_count: 3)
 ]
 
@@ -73,8 +73,8 @@ describe 'UpdateDb' do
     describe 'given decimations' do
       it 'adds decimations to files' do
         schema = Array.new(simple_db)
-        schema << helper.entry('/folder1/f1_1~decim4')
-        schema << helper.entry('/folder1/f1_1~decim16')
+        schema << helper.entry('/folder1/f1_1~decim-4')
+        schema << helper.entry('/folder1/f1_1~decim-16')
         update_with_schema(schema)
         folder1 = @root.subfolders[0]
         file1 = folder1.db_files[0]
@@ -83,7 +83,7 @@ describe 'UpdateDb' do
       it 'ignores orphaned decimations' do
         schema = Array.new(simple_db)
         # no /folder1/f1_3 so this is an orphan decimation
-        schema << helper.entry('/folder1/f1_3~decim4')
+        schema << helper.entry('/folder1/f1_3~decim-4')
         update_with_schema(schema)
         folder1 = @root.subfolders[0]
         # expect just 2 files in this folder

vagrant_boxes/Vagrantfile

@@ -37,7 +37,7 @@ Vagrant.configure("2") do |config|
   # the path on the host to the actual folder. The second argument is
   # the path on the guest to mount the folder. And the optional third
   # argument is a set of non-required options.
-  # config.vm.synced_folder "../data", "/vagrant_data"
+  config.vm.synced_folder "/Users/jdonnal/nilm_repos/control_panel/vagrant_folder", "/vagrant"
 
   # Provider-specific configuration so you can fine-tune various
   # backing providers for Vagrant. These expose provider-specific options.