more specs for db_update

.rspec

 --color
 --color
 --require spec_helper
 --require spec_helper
+#--format documentation

app/adapters/db_adapter.rb

@@ -11,14 +11,18 @@ class DbAdapter
   def schema # rubocop:disable Metrics/MethodLength
   def schema # rubocop:disable Metrics/MethodLength
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump = self.class.get("#{@url}/stream/list?extended=1")
     dump.parsed_response.map do |entry|
     dump.parsed_response.map do |entry|
+      # TODO: implement global metadata dump, because this is sloooow
+      dump = self.class.get("#{@url}/stream/get_metadata?path=#{entry[0]}")
+      metadata = dump.parsed_response['config_key__']
       { path:       entry[0],
       { path:       entry[0],
         attributes: {
         attributes: {
           data_type:  entry[1],
           data_type:  entry[1],
           start_time: entry[2] || 0,
           start_time: entry[2] || 0,
           end_time:   entry[3] || 0,
           end_time:   entry[3] || 0,
           total_rows: entry[4],
           total_rows: entry[4],
-          total_time: entry[5] },
-        metadata: {} }
+          total_time: entry[5]
+        }.merge(metadata)
+      }
     end
     end
   end
   end
 end
 end

app/services/db/update_db.rb

@@ -150,11 +150,15 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
   def __build_file(folder:, entry_group:,
   def __build_file(folder:, entry_group:,
                    default_name:)
                    default_name:)
     base = __base_entry(entry_group)
     base = __base_entry(entry_group)
-    return unless base # corrupt file, don't process
+    unless base # corrupt file, don't process
+      @warnings << "#{entry_group.count} orphan decimations in #{folder.name}"
+      return
+    end
     # find or create the file
     # find or create the file
     file = folder.db_files.find_by_path(base[:path])
     file = folder.db_files.find_by_path(base[:path])
     file ||= DbFile.new(db_folder: folder, path: base[:path])
     file ||= DbFile.new(db_folder: folder, path: base[:path])
     info = { name: default_name }.merge(base[:metadata])
     info = { name: default_name }.merge(base[:metadata])
+    # automatically updates the streams for this file
     file.update_attributes(info)
     file.update_attributes(info)
     file.save!
     file.save!
     __build_decimations(file: file,
     __build_decimations(file: file,
@@ -168,10 +172,7 @@ class UpdateDb # rubocop:disable Metrics/ClassLength
     base_entry = entry_group.select { |entry|
     base_entry = entry_group.select { |entry|
       entry[:path].match(decimation_tag).nil?
       entry[:path].match(decimation_tag).nil?
     }.first
     }.first
-    unless base_entry
-      warnings << "Missing base stream for #{default_name} in #{folder.name}"
-      return nil
-    end
+    return nil unless base_entry
     base_entry
     base_entry
   end
   end
 
 

spec/services/db/update_db_spec.rb

@@ -6,19 +6,21 @@ helper = DbSchemaHelper.new
 # a simple schema that could be returned
 # a simple schema that could be returned
 # from DbAdapater.schema
 # from DbAdapater.schema
 # folder1
 # folder1
-#  `- file1_1
-#   - file1_2
+#  `- file1_1: 4 streams
+#   - file1_2: 5 streams
 # folder2
 # folder2
-#  '- file2_1
-#  `- file2_2
+#  '- file2_1: 1 stream
+#  `- file2_2: 3 streams
 
 
 simple_db = [
 simple_db = [
   helper.entry('/folder1/f1_1',
   helper.entry('/folder1/f1_1',
-               metadata: { name: 'file1_1' }, stream_count: 4),
+               metadata: { name: 'file1_1' }, stream_count: 3),
   helper.entry('/folder1/f1_2',
   helper.entry('/folder1/f1_2',
                metadata: { name: 'file1_2' }, stream_count: 5),
                metadata: { name: 'file1_2' }, stream_count: 5),
-  helper.entry('/folder2/f2_1', metadata: { name: 'file2_1' }),
-  helper.entry('/folder2/f2_2', metadata: { name: 'file2_2' })
+  helper.entry('/folder2/f2_1',
+               metadata: { name: 'file2_1' }, stream_count: 1),
+  helper.entry('/folder2/f2_2', 
+               metadata: { name: 'file2_2' }, stream_count: 3)
 ]
 ]
 
 
 describe 'UpdateDb' do
 describe 'UpdateDb' do
@@ -33,6 +35,7 @@ describe 'UpdateDb' do
       @service.run(db_adapter: adapter)
       @service.run(db_adapter: adapter)
       @root = @db.root_folder
       @root = @db.root_folder
     end
     end
+    # simple schema parsing
     describe 'given the simple_db schema' do
     describe 'given the simple_db schema' do
       it 'builds a root folder' do
       it 'builds a root folder' do
         update_with_schema(simple_db)
         update_with_schema(simple_db)
@@ -56,7 +59,6 @@ describe 'UpdateDb' do
         expect(file1.db_streams.count).to eq(4)
         expect(file1.db_streams.count).to eq(4)
         expect(file2.db_streams.count).to eq(5)
         expect(file2.db_streams.count).to eq(5)
       end
       end
-
       it 'builds sub-folder2' do
       it 'builds sub-folder2' do
         update_with_schema(simple_db)
         update_with_schema(simple_db)
         folder2 = @root.subfolders[1]
         folder2 = @root.subfolders[1]
@@ -67,14 +69,8 @@ describe 'UpdateDb' do
       end
       end
     end
     end
 
 
-    it 'uses folder info streams if available' do
-      schema = Array.new(simple_db)
-      schema << helper.entry('/folder1/info', metadata: { name: 'first' })
-      update_with_schema(schema)
-      folder1 = @root.subfolders[0]
-      expect(folder1.name).to eq('first')
-    end
-
+    # decimation handling
+    describe 'given decimations' do
       it 'adds decimations to files' do
       it 'adds decimations to files' do
         schema = Array.new(simple_db)
         schema = Array.new(simple_db)
         schema << helper.entry('/folder1/f1_1~decim4')
         schema << helper.entry('/folder1/f1_1~decim4')
@@ -84,5 +80,30 @@ describe 'UpdateDb' do
         file1 = folder1.db_files[0]
         file1 = folder1.db_files[0]
         expect(file1.db_decimations.count).to eq(2)
         expect(file1.db_decimations.count).to eq(2)
       end
       end
+      it 'ignores orphaned decimations' do
+        schema = Array.new(simple_db)
+        # no /folder1/f1_3 so this is an orphan decimation
+        schema << helper.entry('/folder1/f1_3~decim4')
+        update_with_schema(schema)
+        folder1 = @root.subfolders[0]
+        # expect just 2 files in this folder
+        expect(folder1.db_files.count).to eq(2)
+        # and a warning about orphaned decimations
+        expect(@service.warnings.count).to eq(1)
+      end
+    end
+
+    # info streams and metadata
+    describe 'given info streams' do
+      it 'uses info stream to set folder data' do
+        schema = Array.new(simple_db)
+        schema << helper.entry('/folder1/info', metadata: { name: 'first' })
+        update_with_schema(schema)
+        folder1 = @root.subfolders[0]
+        expect(folder1.name).to eq('first')
+      end
+      it 'reads metadata from base file'
+      it 'reads metadata from decimations'
+    end
   end
   end
 end
 end