can refresh database

app/adapters/db_adapter.rb

@@ -8,6 +8,16 @@ class DbAdapter
     @url = url
   end
 
+  def dbinfo
+    version = self.class.get("#{@url}/version").parsed_response
+    info = self.class.get("#{@url}/dbinfo").parsed_response
+    {
+      version: version,
+      size_db: info['size'],
+      size_other: info['other'],
+      size_total: info["size"]+info["other"]+info["free"]+info["reserved"]
+    }
+  end
   def schema
     # GET extended info stream list
     dump = self.class.get("#{@url}/stream/list?extended=1")

app/controllers/db_streams_controller.rb

@@ -6,8 +6,8 @@ class DbStreamsController < ApplicationController
     stream = DbStream.find(params[:id])
     adapter = DbAdapter.new(stream.db.url)
     service = EditStream.new(adapter)
-    service.run(stream, stream_params.symbolize_keys)
-    if(service.success?)
+    service.run(stream, stream_params)
+    if service.success?
       render json: stream
     else
       render json: service, status: :unprocessable_entity
@@ -15,8 +15,13 @@ class DbStreamsController < ApplicationController
   end
 
   private
-    def stream_params
-      params.permit(:name, :description, :hidden, :name_abbrev, :elements)
-    end
 
+  def stream_params
+    params.require(:stream)
+          .permit(:name, :description, :name_abbrev, :hidden,
+                  db_elements_attributes:
+                  [:id, :name, :units,
+                   :default_max, :default_min, :scale_factor, :offset,
+                   :plottable, :discrete])
+  end
 end

app/controllers/dbs_controller.rb

@@ -6,4 +6,36 @@ class DbsController < ApplicationController
     db = Db.find(params[:id])
     render json: db
   end
+
+  def update
+    db = Db.find(params[:id])
+    prev_url = db.url
+    if db.update_attributes(db_params)
+      if(prev_url != db.url || params[:refresh])
+        refresh(db) and return
+      end
+      render json: db
+    else
+      render json: "adfs"
+    end
+  end
+
+  private
+
+    def refresh(db)
+      adapter = DbAdapter.new(db.url)
+      service = UpdateDb.new(db: db)
+      service.run(adapter.dbinfo, adapter.schema)
+      if(service.success?)
+        render json: db
+      else
+        render json: service, status: :unprocessable_entity
+      end
+    end
+
+
+    def db_params
+      params.permit(:url, :max_points_per_plot)
+    end
+
 end

app/models/db_element.rb

@@ -9,6 +9,21 @@ class DbElement < ApplicationRecord
   validates :name, uniqueness: { scope: :db_stream_id,
     message: ' is already used in this folder'}
 
+  validates :scale_factor, presence: true, numericality: true
+  validates :scale_factor, presence: true, numericality: true
+  validates :default_min, allow_nil: true, numericality: true
+  validates :default_max, allow_nil: true, numericality: true
+
+  # force set any validated params to acceptable
+  # default values this allows us to process corrupt databases
+  def use_default_attributes
+    self.name = "element#{self.column}"
+    self.units = ""
+    self.default_min = nil
+    self.default_max = nil
+    self.scale_factor = 1.0
+    self.offset = 0.0
+  end
 
   def as_json(_options = {})
     super(except: [:created_at, :updated_at])

app/models/db_folder.rb

@@ -19,11 +19,13 @@ class DbFolder < ApplicationRecord
   validates_presence_of :name
   # validates_with DbFolderValidator
   validates :name, uniqueness: { scope: :parent_id,
-    message: ' is already used in this folder'}
+    message: ' is already used in this folder'}, unless: :root_folder?
 
   #:section: Utility Methods
 
-
+  def root_folder?
+    self.parent == nil
+  end
 
 
   def self.defined_attributes
@@ -38,6 +40,13 @@ class DbFolder < ApplicationRecord
     true
   end
 
+  # force set any validated params to acceptable
+  # default values this allows us to process corrupt databases
+  def use_default_attributes
+    self.name = self.path
+    self.description = ''
+  end
+
   def as_json(options = {shallow: true})
     folder = super(except: [:created_at, :updated_at])
     if(options[:shallow]== false)

app/models/db_stream.rb

@@ -45,6 +45,14 @@ class DbStream < ApplicationRecord
     /^\w*_(\d*)$/.match(data_type)[1].to_i
   end
 
+  # force set any validated params to acceptable
+  # default values this allows us to process corrupt databases
+  def use_default_attributes
+    self.name = self.path
+    self.description = ''
+  end
+
+  
   def as_json(_options = {})
     stream = super(except: [:created_at, :updated_at])
     stream[:elements] = db_elements.map(&:as_json)

app/services/db/update_db.rb

@@ -9,7 +9,7 @@ class UpdateDb
     super()
   end
 
-  def run(schema)
+  def run(dbinfo, schema)
     # create the root folder if it doesn't exist
     @db.root_folder ||= DbFolder.create(db: @db, name: 'root', path: '/')
     @root_folder = @db.root_folder
@@ -18,6 +18,11 @@ class UpdateDb
     entries = __create_entries(schema)
 
     updater = UpdateFolder.new(@root_folder, entries)
+    # update db attributes from dbinfo
+    @db.size_total = dbinfo[:size_total]
+    @db.size_db = dbinfo[:size_db]
+    @db.size_other = dbinfo[:size_other]
+    @db.version = dbinfo[:version]
     absorb_status(updater.run)
 
     @db.save

app/services/db_folder/update_folder.rb

@@ -25,9 +25,12 @@ class UpdateFolder
   def run
     # update the folder attributes from metadata
     info = __read_info_entry(@entries) || {}
-    @folder.update_attributes(
-      info.slice(*DbFolder.defined_attributes)
-    )
+    # if metadata is corrupt, use default values instead
+    unless @folder.update_attributes(
+      info.slice(*DbFolder.defined_attributes))
+      @folder.use_default_attributes
+      Rails.logger.warn("corrupt metadata: #{@folder.path}")
+    end
     # process the contents of the folder
     __parse_folder_entries(@folder, @entries)
     # delete any streams or folders still in the
@@ -48,6 +51,9 @@ class UpdateFolder
     @folder.end_time = @end_time
     @folder.size_on_disk = @size_on_disk
     # save the result
+    unless @folder.valid?
+      byebug
+    end
     @folder.save!
     self
   end
@@ -183,7 +189,6 @@ class UpdateFolder
   # update extents based on result of updater
   # (either a stream or a subfolder)
   def absorb_data_extents(updater)
-    byebug if(@folder.name=="tutorial")
     if @start_time.nil?
       @start_time = updater.start_time
     elsif !updater.start_time.nil?

app/services/db_stream/edit_stream.rb

@@ -11,18 +11,11 @@ class EditStream
 
   def run(db_stream, attribs)
     # only accept valid attributes
-    stream_attribs = attribs.slice(:name, :description,
-                                   :hidden, :name_abbrev)
-    begin
-      stream_attribs[:db_elements_attributes] =
-        __parse_element_attribs(attribs[:elements])
-    rescue TypeError
-      add_error("invalid db_elements_attributes parameter")
-      return self
-    end
+    attribs.slice!(:name, :description, :name_abbrev, :hidden,
+                   :db_elements_attributes)
     # assign the new attributes and check if the
     # result is valid (eg elements can't have the same name)
-    db_stream.assign_attributes(stream_attribs)
+    db_stream.assign_attributes(attribs)
     unless db_stream.valid?
       db_stream.errors
                .full_messages

app/services/db_stream/update_stream.rb

@@ -29,7 +29,12 @@ class UpdateStream
   # create or update a DbStream object at the
   # specified path.
   def __update_stream(stream, base_entry, decimation_entries)
-    stream.update_attributes(base_entry[:attributes])
+    # use default attributes if metadata is corrupt
+    unless stream.update_attributes(base_entry[:attributes])
+      stream.use_default_attributes
+      Rails.logger.warn("corrupt metadata: #{stream.path}")
+
+    end
     __compute_extents([base_entry] + decimation_entries)
     stream.start_time = @start_time
     stream.end_time = @end_time
@@ -58,11 +63,16 @@ class UpdateStream
   def __build_elements(stream:, stream_data:)
     stream.column_count.times do |x|
       element = stream.db_elements.find_by_column(x)
-      element ||= DbElement.new(db_stream: stream)
+      element ||= DbElement.new(db_stream: stream, column: x)
       # check if there is stream metadata for column x
       entry = stream_data.select { |meta| meta[:column] == x }
       # use the metadata if present
-      element.update_attributes(entry[0] || {})
+      unless element.update_attributes(entry[0] || {})
+        element.use_default_attributes
+        Rails.logger.warn(stream_data)
+        Rails.logger.warn("corrupt metadata: #{stream.path}:"\
+                          "e#{element.column}")
+      end
       element.save!
     end
   end

app/services/nilm/create_nilm.rb

@@ -18,6 +18,6 @@ class CreateNilm
     db = Db.create(nilm: @nilm, url: db_url)
     service = UpdateDb.new(db: db)
     adapter = DbAdapter.new(db.url)
-    service.run(adapter.schema)
+    service.run(adapter.dbinfo, adapter.schema)
   end
 end

db/migrate/20170125030038_add_fields_to_db.rb

+class AddFieldsToDb < ActiveRecord::Migration[5.0]
+  def change
+    add_column :dbs, :size_total, :integer, :limit => 8
+    add_column :dbs, :size_db, :integer, :limit => 8
+    add_column :dbs, :size_other, :integer, :limit => 8
+    add_column :dbs, :version, :string
+    add_column :dbs, :max_points_per_plot, :integer, :default => 3600
+  end
+end

db/migrate/20170127033854_extend_size_on_disk_type.rb

+class ExtendSizeOnDiskType < ActiveRecord::Migration[5.0]
+  def change
+    change_column :db_folders, :size_on_disk, :integer, limit: 8
+    change_column :db_streams, :size_on_disk, :integer, limit: 8
+
+
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170118013812) do
+ActiveRecord::Schema.define(version: 20170127033854) do
 
   create_table "db_decimations", force: :cascade do |t|
     t.integer  "start_time",   limit: 8
@@ -50,7 +50,7 @@ ActiveRecord::Schema.define(version: 20170118013812) do
     t.integer  "db_id"
     t.integer  "start_time",   limit: 8
     t.integer  "end_time",     limit: 8
-    t.integer  "size_on_disk"
+    t.integer  "size_on_disk", limit: 8
   end
 
   create_table "db_streams", force: :cascade do |t|
@@ -68,16 +68,21 @@ ActiveRecord::Schema.define(version: 20170118013812) do
     t.string   "name_abbrev"
     t.boolean  "delete_locked"
     t.boolean  "hidden"
-    t.integer  "size_on_disk"
+    t.integer  "size_on_disk",  limit: 8
     t.integer  "db_id"
   end
 
   create_table "dbs", force: :cascade do |t|
     t.string   "url"
     t.integer  "db_folder_id"
-    t.datetime "created_at",   null: false
-    t.datetime "updated_at",   null: false
+    t.datetime "created_at",                                   null: false
+    t.datetime "updated_at",                                   null: false
     t.integer  "nilm_id"
+    t.integer  "size_total",          limit: 8
+    t.integer  "size_db",             limit: 8
+    t.integer  "size_other",          limit: 8
+    t.string   "version"
+    t.integer  "max_points_per_plot",           default: 3600
   end
 
   create_table "nilms", force: :cascade do |t|

spec/models/db_spec.rb

@@ -6,6 +6,13 @@ RSpec.describe 'Db' do
     let(:db) { Db.new }
     specify { expect(db).to respond_to(:url) }
     specify { expect(db).to respond_to(:root_folder) }
+    
+    specify { expect(db).to respond_to(:size_db) }
+    specify { expect(db).to respond_to(:size_total) }
+    specify { expect(db).to respond_to(:size_other) }
+    specify { expect(db).to respond_to(:version) }
+    specify { expect(db).to respond_to(:max_points_per_plot) }
+
   end
 
   it 'removes the root folder when destroyed' do

spec/services/db/update_db_spec.rb

@@ -24,11 +24,13 @@ simple_db = [
 ]
 
 describe 'UpdateDb' do
+  let(:dbinfo) { double('dbinfo').as_null_object}
   describe '*run*' do
     def update_with_schema(schema, db: nil)
       @db = db || Db.new
       @service = UpdateDb.new(db: @db)
-      @service.run(schema)
+      mock_info =
+      @service.run(dbinfo, schema) #ignore dbinfo
       @root = @db.root_folder
     end
     # simple schema parsing

spec/services/db_stream/edit_stream_spec.rb

@@ -14,7 +14,7 @@ describe 'EditStream service' do
   it 'changes stream and element attributes' do
     attribs = { id: 0, invalid_attrib: 'ignore',
                 name: 'new name', name_abbrev: 'nn',
-              elements: [{id: element.id, name: 'new!'}] }
+              db_elements_attributes: [{id: element.id, name: 'new!'}] }
     allow(db_adapter).to receive(:set_stream_metadata).and_return(success)
     # run the service, it should call the adapter and save the folder
     service.run(stream, attribs)
@@ -34,7 +34,8 @@ describe 'EditStream service' do
   end
 
   it 'does not change stream or elements on a server error' do
-    attribs = { name: 'new', elements: [{id: element.id, name: 'new'}]}
+    attribs = { name: 'new',
+                db_elements_attributes: [{id: element.id, name: 'new'}]}
     allow(db_adapter).to receive(:set_stream_metadata).and_return(error)
     allow(stream).to receive(:save!)
     allow(element).to receive(:save!)
@@ -45,12 +46,4 @@ describe 'EditStream service' do
     expect(element).to_not have_received(:save!)
   end
 
-  it 'produces error for invalid parameter structure' do
-    attribs = {name: 'new', elements: {id: 0, name: 'nice try'}}
-    allow(db_adapter).to receive(:set_stream_metadata).and_return(success)
-    # run the service, it shouldn't call the database adapter
-    service.run(stream, attribs)
-    expect(service.errors?).to be true
-    expect(db_adapter).to_not have_received(:set_stream_metadata)
-  end
 end

spec/services/db_stream/update_stream_spec.rb

@@ -6,6 +6,8 @@ describe 'UpdateStream service' do
   let(:db) { Db.new }
   let(:service) { UpdateDb.new(db: db) }
   let(:helper) { DbSchemaHelper.new }
+  let(:mock_dbinfo) { double('dbinfo').as_null_object}
+
   def build_entry(path, start, last, rows, width)
     helper.entry(path, start_time: start, end_time: last,
                  element_count: width, total_rows: rows)
@@ -14,13 +16,13 @@ describe 'UpdateStream service' do
 
   it 'updates stream info' do
     # create Db with 1 folder and stream
-    service.run([helper.entry('/folder1/stream1',
+    service.run(mock_dbinfo, [helper.entry('/folder1/stream1',
                               metadata: { name: 'old_name' })])
     stream = DbStream.find_by_name('old_name')
     expect(stream).to be_present
     # run update again with new metadata
     service = UpdateDb.new(db: db)
-    service.run([helper.entry('/folder1/stream1',
+    service.run(mock_dbinfo, [helper.entry('/folder1/stream1',
                               metadata: { name: 'new_name' })])
     stream.reload
     expect(stream.name).to eq('new_name')
@@ -32,7 +34,7 @@ describe 'UpdateStream service' do
     # expect stream to have min_start => max_end duration
     # and size_on_disk to be sum of base+decimations
 
-    service.run([build_entry('/a/path',            1,  90, 20, 8),
+    service.run(mock_dbinfo,[build_entry('/a/path',            1,  90, 20, 8),
                  build_entry('/a/path~decim-4',   10, 110, 25, 24),
                  build_entry('/a/path~decim-16', -10, 100, 28, 24),
                  build_entry('/a/path~decim-64', nil, nil, 0, 24)])
@@ -48,14 +50,26 @@ describe 'UpdateStream service' do
     schema = [helper.entry('/folder1/subfolder/stream',
                            element_count: 1)]
     schema[0][:elements][0][:name] = 'old_name'
-    service.run(schema)
+    service.run(mock_dbinfo, schema)
     element = DbElement.find_by_name('old_name')
     expect(element).to be_present
     # run update again with new metadata
     schema[0][:elements][0][:name] = 'new_name'
     service = UpdateDb.new(db: db)
-    service.run(schema)
+    service.run(mock_dbinfo, schema)
     element.reload
     expect(element.name).to eq('new_name')
   end
+
+  it 'uses default attributes if metadata is corrupt' do
+    #metadata missing stream name and an element name
+    bad_entry = helper.entry('/a/path',metadata: {name: ''})
+    bad_entry[:elements] = [{column: 0, name: ''}]
+    service.run(mock_dbinfo, [bad_entry])
+    stream = DbStream.find_by_path('/a/path')
+    expect(stream.name).to eq('/a/path')
+    expect(stream.db_elements
+                 .find_by_column(0)
+                 .name).to eq('element0')
+  end
 end

spec/services/folder/edit_folder_spec.rb

@@ -40,6 +40,5 @@ describe 'EditFolder service' do
     service.run(folder, attribs)
     expect(service.errors?).to be true
     expect(folder).to_not have_received(:save!)
-
   end
 end

spec/services/folder/update_folder_spec.rb

@@ -5,22 +5,24 @@ require 'rails_helper'
 describe 'UpdateFolder service' do
   let(:db) { Db.new }
   let(:service) { UpdateDb.new(db: db) }
-  let(:helper) {DbSchemaHelper.new}
+  let(:helper) { DbSchemaHelper.new }
+  let(:mock_dbinfo) { double('dbinfo').as_null_object }
+
   def build_entry(path, start, last, rows, width)
     helper.entry(path, start_time: start, end_time: last,
-                 element_count: width, total_rows: rows)
+                       element_count: width, total_rows: rows)
   end
 
   it 'updates folder info' do
     # create Db with folder and subfolder
-    service.run([helper.entry('/folder1/subfolder/info',
-                              metadata: { name: 'old_name' })])
+    service.run(mock_dbinfo, [helper.entry('/folder1/subfolder/info',
+                                           metadata: { name: 'old_name' })])
     folder = DbFolder.find_by_name('old_name')
     expect(folder).to be_present
     # run update again with new metadata
     service = UpdateDb.new(db: db)
-    service.run([helper.entry('/folder1/subfolder/info',
-                              metadata: { name: 'new_name' })])
+    service.run(mock_dbinfo, [helper.entry('/folder1/subfolder/info',
+                                           metadata: { name: 'new_name' })])
     folder.reload
     expect(folder.name).to eq('new_name')
     expect(folder.db).to eq(db)
@@ -31,15 +33,30 @@ describe 'UpdateFolder service' do
     # expect stream to have min_start => max_end duration
     # and size_on_disk to be sum of base+decimations
 
-    service.run([build_entry('/a/path',            1,  90, 20, 8),
-                 build_entry('/a/path~decim-4',   10, 110, 25, 24),
-                 build_entry('/a/path2',         -10, 100, 28, 24),
-                 #build_entry('/a/path',       nil, nil, 0, 24),
-                 build_entry('/a/deep/path',       0, 400, 8, 10)])
+    service.run(mock_dbinfo, [build_entry('/a/path', 1, 90, 20, 8),
+                              build_entry('/a/path~decim-4', 10, 110, 25, 24),
+                              build_entry('/a/path2', -10, 100, 28, 24),
+                              # build_entry('/a/path',       nil, nil, 0, 24),
+                              build_entry('/a/deep/path',       0, 400, 8, 10)])
     folder = DbFolder.find_by_path('/a')
     expect(folder.start_time).to eq(-10)
     expect(folder.end_time).to eq(400)
     # (4*8+8)*20 + (4*24+8)*25 + (4*24+8)*28 + (4*10+8)*8
     expect(folder.size_on_disk).to eq(6696)
   end
+
+  it 'uses default attributes if metadata is missing' do
+    service.run(mock_dbinfo, [helper.entry('/a/path', metadata: {})])
+    folder = DbFolder.find_by_path('/a')
+    expect(folder.name).to eq('a')
+  end
+
+  it 'uses default attributes if metadata is corrupt' do
+    bad = { name: '' }
+    service.run(mock_dbinfo, [helper.entry('/a/path'),
+                              helper.entry('/a/info', metadata: bad)])
+
+    folder = DbFolder.find_by_path('/a')
+    expect(folder.name).to eq('/a')
+  end
 end

spec/services/nilm/create_nilm_spec.rb

 # frozen_string_literal: true
 require 'rails_helper'
 
-test_nilm_url = 'http://192.168.42.17'
+test_nilm_url = 'http://localhost:8080'
 
 RSpec.describe 'CreateNilm' do
   describe 'build' do