added size and extents statistics to folders

app/controllers/db_folders_controller.rb

@@ -7,8 +7,6 @@ class DbFoldersController < ApplicationController
     render json: folder, shallow: false
     render json: folder, shallow: false
   end
   end
 
 
-#TODO: add db attribute to folders and streams
-#TODO: add timespan and disk usage stats to folders
 #TODO: create info stream on folders on edit
 #TODO: create info stream on folders on edit
   def update
   def update
     folder = DbFolder.find(params[:id])
     folder = DbFolder.find(params[:id])

app/models/db_folder.rb

@@ -4,6 +4,8 @@
 # and one or more DbFolders as subfolders
 # and one or more DbFolders as subfolders
 class DbFolder < ActiveRecord::Base
 class DbFolder < ActiveRecord::Base
   belongs_to :parent, class_name: 'DbFolder'
   belongs_to :parent, class_name: 'DbFolder'
+  belongs_to :db
+
   has_many :subfolders,
   has_many :subfolders,
            class_name: 'DbFolder',
            class_name: 'DbFolder',
            foreign_key: 'parent_id',
            foreign_key: 'parent_id',

app/models/db_stream.rb

@@ -17,6 +17,8 @@ end
 # A file in the database, contains one or more Streams
 # A file in the database, contains one or more Streams
 class DbStream < ActiveRecord::Base
 class DbStream < ActiveRecord::Base
   belongs_to :db_folder
   belongs_to :db_folder
+  belongs_to :db
+
   has_many :db_elements, dependent: :destroy
   has_many :db_elements, dependent: :destroy
   has_many :db_decimations, dependent: :destroy
   has_many :db_decimations, dependent: :destroy
 
 

app/services/db/update_db.rb

@@ -11,7 +11,7 @@ class UpdateDb
 
 
   def run(schema)
   def run(schema)
     # create the root folder if it doesn't exist
     # create the root folder if it doesn't exist
-    @db.root_folder ||= DbFolder.create(name: 'root', path: '/')
+    @db.root_folder ||= DbFolder.create(db: @db, name: 'root', path: '/')
     @root_folder = @db.root_folder
     @root_folder = @db.root_folder
 
 
     # create the entry array from the schema
     # create the entry array from the schema

app/services/db_folder/update_folder.rb

@@ -3,10 +3,15 @@
 # Handles construction of DbFolder objects
 # Handles construction of DbFolder objects
 class UpdateFolder
 class UpdateFolder
   include ServiceStatus
   include ServiceStatus
+  attr_reader :start_time, :end_time, :size_on_disk
 
 
   def initialize(folder, entries)
   def initialize(folder, entries)
     @folder = folder
     @folder = folder
     @entries = entries
     @entries = entries
+    # initialize extents, these are updated as folders/streams are added
+    @start_time = nil
+    @end_time = nil
+    @size_on_disk = 0
     # initialiaze array of current entries, ids are removed
     # initialiaze array of current entries, ids are removed
     # as they are updated, so any id's left in this
     # as they are updated, so any id's left in this
     # array are no longer present on the remote db
     # array are no longer present on the remote db
@@ -38,7 +43,10 @@ class UpdateFolder
       @folder.subfolders.destroy(*@subfolder_ids)
       @folder.subfolders.destroy(*@subfolder_ids)
       add_warning('Removed folders no longer in the remote database')
       add_warning('Removed folders no longer in the remote database')
     end
     end
-
+    # add the extents computed during updates
+    @folder.start_time = @start_time
+    @folder.end_time = @end_time
+    @folder.size_on_disk = @size_on_disk
     # save the result
     # save the result
     @folder.save!
     @folder.save!
     self
     self
@@ -107,6 +115,7 @@ class UpdateFolder
         updater = __build_folder(folder, entry_group, name)
         updater = __build_folder(folder, entry_group, name)
       end
       end
       absorb_status(updater.run)
       absorb_status(updater.run)
+      absorb_data_extents(updater) # update start, end & size_on_disk
     end
     end
   end
   end
 
 
@@ -129,7 +138,7 @@ class UpdateFolder
     end
     end
     # find or create the stream
     # find or create the stream
     stream = folder.db_streams.find_by_path(base[:path])
     stream = folder.db_streams.find_by_path(base[:path])
-    stream ||= DbStream.new(db_folder: folder,
+    stream ||= DbStream.new(db: folder.db, db_folder: folder,
                             path: base[:path], name: default_name)
                             path: base[:path], name: default_name)
     # remove the id (if present) to mark this stream as updated
     # remove the id (if present) to mark this stream as updated
     @stream_ids -= [stream.id]
     @stream_ids -= [stream.id]
@@ -153,7 +162,8 @@ class UpdateFolder
   def __build_folder(parent, entries, default_name)
   def __build_folder(parent, entries, default_name)
     path = __build_path(entries)
     path = __build_path(entries)
     folder = parent.subfolders.find_by_path(path)
     folder = parent.subfolders.find_by_path(path)
-    folder ||= DbFolder.new(parent: parent, path: path, name: default_name)
+    folder ||= DbFolder.new(parent: parent, db: parent.db,
+                            path: path, name: default_name)
     # remove the id (if present) to mark this folder as updated
     # remove the id (if present) to mark this folder as updated
     @subfolder_ids -= [folder.id]
     @subfolder_ids -= [folder.id]
     # return the Updater, don't run it
     # return the Updater, don't run it
@@ -169,4 +179,20 @@ class UpdateFolder
     parts.pop(entries[0][:chunks].length)
     parts.pop(entries[0][:chunks].length)
     parts.join('/') # stitch parts together to form a path
     parts.join('/') # stitch parts together to form a path
   end
   end
+
+  # update extents based on result of updater
+  # (either a stream or a subfolder)
+  def absorb_data_extents(updater)
+    @start_time = if @start_time.nil?
+                    updater.start_time
+                  else
+                    [@start_time, updater.start_time].min
+                  end
+    @end_time = if @end_time.nil?
+                    updater.end_time
+                  else
+                    [@start_time, updater.end_time].max
+                  end
+    @size_on_disk += updater.size_on_disk
+  end
 end
 end

app/services/db_stream/update_stream.rb

@@ -3,11 +3,16 @@
 # Handles construction of DbFolder objects
 # Handles construction of DbFolder objects
 class UpdateStream
 class UpdateStream
   include ServiceStatus
   include ServiceStatus
+  attr_reader :start_time, :end_time, :size_on_disk
 
 
   def initialize(stream, base_entry, decimation_entries)
   def initialize(stream, base_entry, decimation_entries)
     @stream = stream
     @stream = stream
     @base_entry = base_entry
     @base_entry = base_entry
     @decimation_entries = decimation_entries
     @decimation_entries = decimation_entries
+    # initialize extents, these set during run
+    @start_time = nil
+    @end_time = nil
+    @size_on_disk = 0
     super()
     super()
   end
   end
 
 
@@ -25,7 +30,12 @@ class UpdateStream
   # specified path.
   # specified path.
   def __update_stream(stream, base_entry, decimation_entries)
   def __update_stream(stream, base_entry, decimation_entries)
     stream.update_attributes(base_entry[:attributes])
     stream.update_attributes(base_entry[:attributes])
+    __compute_extents([base_entry] + decimation_entries)
+    stream.start_time = @start_time
+    stream.end_time = @end_time
+    stream.size_on_disk = @size_on_disk
     stream.save!
     stream.save!
+
     __build_decimations(stream: stream,
     __build_decimations(stream: stream,
                         entry_group: decimation_entries)
                         entry_group: decimation_entries)
     __build_elements(stream: stream, stream_data: base_entry[:elements])
     __build_elements(stream: stream, stream_data: base_entry[:elements])
@@ -56,4 +66,30 @@ class UpdateStream
       element.save!
       element.save!
     end
     end
   end
   end
+
+  # compute the time range and total size of this stream
+  # accepts an array of entries (include base & decim)
+  def __compute_extents(entries)
+    entries.map { |x| x[:attributes] }.each do |attrs|
+      next if (attrs[:total_rows]).zero?
+      if @start_time.nil?
+        @start_time = attrs[:start_time]
+        @end_time = attrs[:end_time]
+      end
+      @start_time = [@start_time, attrs[:start_time]].min
+      @end_time = [@end_time, attrs[:end_time]].max
+      @size_on_disk += attrs[:total_rows] *
+                       __bytes_per_row(attrs[:data_type])
+    end
+  end
+
+  # compute how many bytes are required per row based
+  # on the datatype (float32_8 => 4*8+8)
+  def __bytes_per_row(data_type)
+    regex = /[a-z]*(\d*)_(\d*)/.match(data_type)
+    dtype_bytes = regex[1].to_i / 8
+    num_cols = regex[2].to_i
+    ts_bytes = 8
+    ts_bytes + num_cols * dtype_bytes
+  end
 end
 end

db/migrate/20170118011117_add_items_to_db_folders.rb

+class AddItemsToDbFolders < ActiveRecord::Migration
+  def change
+    add_column :db_folders, :db_id, :integer
+    add_column :db_folders, :start_time, :integer, limit: 8
+    add_column :db_folders, :end_ime, :integer, limit: 8
+    add_column :db_folders, :size_on_disk, :integer
+
+    add_column :db_streams, :size_on_disk, :integer
+  end
+end

db/migrate/20170118011803_fix_db_folder_attr.rb

+class FixDbFolderAttr < ActiveRecord::Migration
+  def change
+    rename_column :db_folders, :end_ime, :end_time
+  end
+end

db/migrate/20170118013812_add_db_to_db_stream.rb

+class AddDbToDbStream < ActiveRecord::Migration
+  def change
+    add_column :db_streams, :db_id, :integer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 20170104213820) do
+ActiveRecord::Schema.define(version: 20170118013812) do
 
 
   create_table "db_decimations", force: :cascade do |t|
   create_table "db_decimations", force: :cascade do |t|
     t.integer  "start_time",   limit: 8
     t.integer  "start_time",   limit: 8
@@ -48,6 +48,10 @@ ActiveRecord::Schema.define(version: 20170104213820) do
     t.integer  "parent_id"
     t.integer  "parent_id"
     t.string   "path"
     t.string   "path"
     t.boolean  "hidden"
     t.boolean  "hidden"
+    t.integer  "db_id"
+    t.integer  "start_time",   limit: 8
+    t.integer  "end_time",     limit: 8
+    t.integer  "size_on_disk"
   end
   end
 
 
   create_table "db_streams", force: :cascade do |t|
   create_table "db_streams", force: :cascade do |t|
@@ -65,6 +69,8 @@ ActiveRecord::Schema.define(version: 20170104213820) do
     t.string   "name_abbrev"
     t.string   "name_abbrev"
     t.boolean  "delete_locked"
     t.boolean  "delete_locked"
     t.boolean  "hidden"
     t.boolean  "hidden"
+    t.integer  "size_on_disk"
+    t.integer  "db_id"
   end
   end
 
 
   create_table "dbs", force: :cascade do |t|
   create_table "dbs", force: :cascade do |t|

spec/factories/db_schema_helper.rb

@@ -4,14 +4,15 @@
 # are usually returned by DbAdapter.schema
 # are usually returned by DbAdapter.schema
 class DbSchemaHelper
 class DbSchemaHelper
   # schema data
   # schema data
-  def entry(path, metadata: {}, element_count: 1)
+  def entry(path, metadata: {}, element_count: 1,
+    start_time: nil, end_time: nil, total_rows: 0)
     {
     {
       path: path,
       path: path,
       attributes: {
       attributes: {
         data_type: "float32_#{element_count}",
         data_type: "float32_#{element_count}",
-        start_time: 0,
-        end_time: 0,
-        total_rows: 0,
+        start_time: start_time,
+        end_time: end_time,
+        total_rows: total_rows,
         total_time: 0
         total_time: 0
       }.merge(metadata),
       }.merge(metadata),
       elements: __build_elements(element_count)
       elements: __build_elements(element_count)

spec/models/db_folder_spec.rb

@@ -4,12 +4,21 @@ require 'rails_helper'
 RSpec.describe 'DbFolder' do
 RSpec.describe 'DbFolder' do
   describe 'object' do
   describe 'object' do
     let(:db_folder) { DbFolder.new }
     let(:db_folder) { DbFolder.new }
+    # attributes
     specify { expect(db_folder).to respond_to(:name) }
     specify { expect(db_folder).to respond_to(:name) }
     specify { expect(db_folder).to respond_to(:description) }
     specify { expect(db_folder).to respond_to(:description) }
+    specify { expect(db_folder).to respond_to(:hidden) }
+    specify { expect(db_folder).to respond_to(:start_time) }
+    specify { expect(db_folder).to respond_to(:end_time) }
+    specify { expect(db_folder).to respond_to(:size_on_disk) }
+    # associations
     specify { expect(db_folder).to respond_to(:parent) }
     specify { expect(db_folder).to respond_to(:parent) }
     specify { expect(db_folder).to respond_to(:subfolders) }
     specify { expect(db_folder).to respond_to(:subfolders) }
     specify { expect(db_folder).to respond_to(:db_streams) }
     specify { expect(db_folder).to respond_to(:db_streams) }
-    specify { expect(db_folder).to respond_to(:hidden) }
+    specify { expect(db_folder).to respond_to(:db) }
+
+
+
   end
   end
 
 
   describe 'when destroyed' do
   describe 'when destroyed' do

spec/models/db_stream_spec.rb

@@ -4,11 +4,17 @@ require 'rails_helper'
 RSpec.describe 'DbStream' do
 RSpec.describe 'DbStream' do
   describe 'object' do
   describe 'object' do
     let(:db_stream) { DbStream.new }
     let(:db_stream) { DbStream.new }
+    # attributes
     specify { expect(db_stream).to respond_to(:name) }
     specify { expect(db_stream).to respond_to(:name) }
     specify { expect(db_stream).to respond_to(:name_abbrev) }
     specify { expect(db_stream).to respond_to(:name_abbrev) }
     specify { expect(db_stream).to respond_to(:description) }
     specify { expect(db_stream).to respond_to(:description) }
-    specify { expect(db_stream).to respond_to(:db_elements) }
     specify { expect(db_stream).to respond_to(:hidden) }
     specify { expect(db_stream).to respond_to(:hidden) }
+    specify { expect(db_stream).to respond_to(:size_on_disk) }
+    # associations
+    specify { expect(db_stream).to respond_to(:db_elements) }
+    specify { expect(db_stream).to respond_to(:db) }
+
+
   end
   end
 
 
   describe 'child elements' do
   describe 'child elements' do

spec/services/db_stream/update_stream_spec.rb

 # frozen_string_literal: true
 # frozen_string_literal: true
 
 
 require 'rails_helper'
 require 'rails_helper'
-helper = DbSchemaHelper.new
 
 
 describe 'UpdateStream service' do
 describe 'UpdateStream service' do
   let(:db) { Db.new }
   let(:db) { Db.new }
   let(:service) { UpdateDb.new(db: db) }
   let(:service) { UpdateDb.new(db: db) }
+  let(:helper) { DbSchemaHelper.new }
+  def build_entry(path, start, last, rows, width)
+    helper.entry(path, start_time: start, end_time: last,
+                 element_count: width, total_rows: rows)
+  end
+
 
 
   it 'updates stream info' do
   it 'updates stream info' do
     # create Db with 1 folder and stream
     # create Db with 1 folder and stream
@@ -19,6 +24,23 @@ describe 'UpdateStream service' do
                               metadata: { name: 'new_name' })])
                               metadata: { name: 'new_name' })])
     stream.reload
     stream.reload
     expect(stream.name).to eq('new_name')
     expect(stream.name).to eq('new_name')
+    expect(stream.db).to eq(db)
+  end
+
+  it 'updates extent info' do
+    # create a stream with 2 decimations
+    # expect stream to have min_start => max_end duration
+    # and size_on_disk to be sum of base+decimations
+
+    service.run([build_entry('/a/path',            1,  90, 20, 8),
+                 build_entry('/a/path~decim-4',   10, 110, 25, 24),
+                 build_entry('/a/path~decim-16', -10, 100, 28, 24),
+                 build_entry('/a/path~decim-64', nil, nil, 0, 24)])
+    stream = DbStream.find_by_path('/a/path')
+    expect(stream.start_time).to eq(-10)
+    expect(stream.end_time).to eq(110)
+    # (4*8+8)*20 + (4*24+8)*25 + (4*24+8)*28 + nothing
+    expect(stream.size_on_disk).to eq(6312)
   end
   end
   
   
   it 'updates element info' do
   it 'updates element info' do

spec/services/folder/update_folder_spec.rb

 # frozen_string_literal: true
 # frozen_string_literal: true
 
 
 require 'rails_helper'
 require 'rails_helper'
-helper = DbSchemaHelper.new
 
 
 describe 'UpdateFolder service' do
 describe 'UpdateFolder service' do
   let(:db) { Db.new }
   let(:db) { Db.new }
   let(:service) { UpdateDb.new(db: db) }
   let(:service) { UpdateDb.new(db: db) }
+  let(:helper) {DbSchemaHelper.new}
+  def build_entry(path, start, last, rows, width)
+    helper.entry(path, start_time: start, end_time: last,
+                 element_count: width, total_rows: rows)
+  end
 
 
   it 'updates folder info' do
   it 'updates folder info' do
     # create Db with folder and subfolder
     # create Db with folder and subfolder
@@ -19,5 +23,22 @@ describe 'UpdateFolder service' do
                               metadata: { name: 'new_name' })])
                               metadata: { name: 'new_name' })])
     folder.reload
     folder.reload
     expect(folder.name).to eq('new_name')
     expect(folder.name).to eq('new_name')
+    expect(folder.db).to eq(db)
+  end
+
+  it 'updates extent info' do
+    # create a stream with 2 decimations
+    # expect stream to have min_start => max_end duration
+    # and size_on_disk to be sum of base+decimations
+
+    service.run([build_entry('/a/path',            1,  90, 20, 8),
+                 build_entry('/a/path~decim-4',   10, 110, 25, 24),
+                 build_entry('/a/path2',         -10, 100, 28, 24),
+                 build_entry('/a/deep/path',       0, 400, 8, 10)])
+    folder = DbFolder.find_by_path('/a')
+    expect(folder.start_time).to eq(-10)
+    expect(folder.end_time).to eq(400)
+    # (4*8+8)*20 + (4*24+8)*25 + (4*24+8)*28 + (4*10+8)*8
+    expect(folder.size_on_disk).to eq(6696)
   end
   end
 end
 end