Skip to content

wattsworth / lumen-api

Go to a project
Commit e117d669 by source_reader
Options

incremental database updates

parent 95794796
Expand all
Inline Side-by-side
Showing with 1766 additions and 30 deletions
app/adapters/joule/backend.rb
...@@ -53,7 +53,7 @@ module Joule ...@@ -53,7 +53,7 @@ module Joule
def db_schema def db_schema
begin begin
resp = self.class.get("#{@url}/streams.json") resp = self.class.get("#{@url}/folders.json")
return nil unless resp.success? return nil unless resp.success?
rescue StandardError => e rescue StandardError => e
Rails.logger.warn "Error retrieving db_schema for #{@url}: [#{e}]" Rails.logger.warn "Error retrieving db_schema for #{@url}: [#{e}]"
......
app/adapters/joule/update_db.rb
...@@ -6,10 +6,17 @@ module Joule ...@@ -6,10 +6,17 @@ module Joule
def initialize(db) def initialize(db)
@db = db @db = db
@deleted_folders = []
@deleted_db_streams = []
@deleted_event_streams = []
super() super()
end end
def run(dbinfo, schema) def run(dbinfo, schema)
# reset the accumulator arrays
@deleted_event_streams = []
@deleted_db_streams = []
@deleted_folders = []
# check to make sure dbinfo and schema are set # check to make sure dbinfo and schema are set
# if either is nil, the database is not available # if either is nil, the database is not available
if dbinfo.nil? || schema.nil? if dbinfo.nil? || schema.nil?
...@@ -20,6 +27,10 @@ module Joule ...@@ -20,6 +27,10 @@ module Joule
# go through the schema and update the database # go through the schema and update the database
@db.root_folder ||= DbFolder.create(db: @db) @db.root_folder ||= DbFolder.create(db: @db)
__update_folder(@db.root_folder, schema, '') __update_folder(@db.root_folder, schema, '')
DbFolder.destroy(@deleted_folders)
DbStream.destroy(@deleted_db_streams)
EventStream.destroy(@deleted_event_streams)
@db.available = true @db.available = true
@db.save @db.save
self self
...@@ -27,6 +38,17 @@ module Joule ...@@ -27,6 +38,17 @@ module Joule
def __update_folder(db_folder, schema, parent_path) def __update_folder(db_folder, schema, parent_path)
attrs = schema.slice(*DbFolder.defined_attributes) attrs = schema.slice(*DbFolder.defined_attributes)
# check to see if this db_folder has changed since last update
attrs[:last_update] = schema[:updated_at].to_datetime
if db_folder.last_update >= attrs[:last_update]
return puts "ignoring #{db_folder.name}:#{db_folder.id}, #{db_folder.last_update}>#{schema[:updated_at]} "
end
if db_folder.id.nil?
puts "creating #{schema[:name]}"
else
puts "updating #{db_folder.name}:#{db_folder.id}"
end
# add in extra attributes that require conversion # add in extra attributes that require conversion
if db_folder.parent.nil? if db_folder.parent.nil?
attrs[:path] = "" attrs[:path] = ""
...@@ -45,9 +67,17 @@ module Joule ...@@ -45,9 +67,17 @@ module Joule
locked = false locked = false
schema[:children].each do |child_schema| schema[:children].each do |child_schema|
child = db_folder.subfolders.find_by_joule_id(child_schema[:id]) child = db_folder.subfolders.find_by_joule_id(child_schema[:id])
if child.nil? # check to see if this folder has been moved from a different location
child = DbFolder.find_by_joule_id(child_schema[:id])
if not child.nil?
child.parent = db_folder
puts "moved #{child.name} to #{db_folder.name}"
@deleted_folders = @deleted_folders - [child.id]
end
end
child ||= DbFolder.new(parent: db_folder, db: db_folder.db) child ||= DbFolder.new(parent: db_folder, db: db_folder.db)
__update_folder(child, child_schema, db_folder.path) __update_folder(child, child_schema, db_folder.path)
size_on_disk+=child.size_on_disk size_on_disk+=child.size_on_disk unless child.size_on_disk.nil?
unless child.start_time.nil? unless child.start_time.nil?
if start_time.nil? if start_time.nil?
start_time = child.start_time start_time = child.start_time
...@@ -65,16 +95,24 @@ module Joule ...@@ -65,16 +95,24 @@ module Joule
updated_ids << child_schema[:id] updated_ids << child_schema[:id]
locked = true if child.locked? locked = true if child.locked?
end end
# remove any subfolders that are no longer on the folder # mark any subfolders that are no longer on the folder for deletion
db_folder.subfolders.where.not(joule_id: updated_ids).destroy_all @deleted_folders += db_folder.subfolders.where.not(joule_id: updated_ids).pluck(:id)
# update or create data streams # update or create data streams
updated_ids=[] updated_ids=[]
schema[:streams].each do |stream_schema| schema[:streams].each do |stream_schema|
stream = db_folder.db_streams.find_by_joule_id(stream_schema[:id]) stream = db_folder.db_streams.find_by_joule_id(stream_schema[:id])
if stream.nil? # check to see if this stream has been moved from a different location
stream = DbStream.find_by_joule_id(stream_schema[:id])
if not stream.nil?
stream.db_folder = db_folder
puts "moved #{stream.name} to #{db_folder.name}"
@deleted_db_streams = @deleted_db_streams - [stream.id]
end
end
stream ||= DbStream.new(db_folder: db_folder, db: db_folder.db) stream ||= DbStream.new(db_folder: db_folder, db: db_folder.db)
__update_stream(stream, stream_schema, db_folder.path) __update_stream(stream, stream_schema, db_folder.path)
size_on_disk+=stream.size_on_disk size_on_disk+=stream.size_on_disk unless stream.size_on_disk.nil?
unless stream.start_time.nil? unless stream.start_time.nil?
if start_time.nil? if start_time.nil?
start_time = stream.start_time start_time = stream.start_time
...@@ -92,20 +130,30 @@ module Joule ...@@ -92,20 +130,30 @@ module Joule
locked=true if stream.locked? locked=true if stream.locked?
updated_ids << stream_schema[:id] updated_ids << stream_schema[:id]
end end
# remove any streams that are no longer in the folder # mark any db streams that are no longer on the folder for deletion
db_folder.db_streams.where.not(joule_id: updated_ids).destroy_all @deleted_db_streams += db_folder.db_streams.where.not(joule_id: updated_ids).pluck(:id)
# update or create event streams # update or create event streams
updated_ids=[] updated_ids=[]
schema[:event_streams] ||= [] schema[:event_streams] ||= []
schema[:event_streams].each do |stream_schema| schema[:event_streams].each do |stream_schema|
stream = db_folder.event_streams.find_by_joule_id(stream_schema[:id]) stream = db_folder.event_streams.find_by_joule_id(stream_schema[:id])
if stream.nil? # check to see if this stream has been moved from a different location
stream = EventStream.find_by_joule_id(stream_schema[:id])
if not stream.nil?
stream.db_folder = db_folder
puts "moved #{stream.name} to #{db_folder.name}"
@deleted_event_streams = @deleted_event_streams - [stream.id]
end
end
stream ||= EventStream.new(db_folder: db_folder, db: db_folder.db) stream ||= EventStream.new(db_folder: db_folder, db: db_folder.db)
__update_event_stream(stream, stream_schema, db_folder.path) __update_event_stream(stream, stream_schema, db_folder.path)
updated_ids << stream_schema[:id] updated_ids << stream_schema[:id]
end end
# remove any streams that are no longer in the folder
db_folder.event_streams.where.not(joule_id: updated_ids).destroy_all # mark any event streams that are no longer in the folder for deletion
@deleted_event_streams += db_folder.event_streams.where.not(joule_id: updated_ids).pluck(:id)
# save the new disk size # save the new disk size
db_folder.size_on_disk = size_on_disk db_folder.size_on_disk = size_on_disk
...@@ -121,13 +169,14 @@ module Joule ...@@ -121,13 +169,14 @@ module Joule
attrs[:path] = "#{parent_path}/#{schema[:name]}" attrs[:path] = "#{parent_path}/#{schema[:name]}"
attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}" attrs[:data_type] = "#{schema[:datatype].downcase}_#{schema[:elements].count}"
attrs[:joule_id] = schema[:id] attrs[:joule_id] = schema[:id]
attrs[:start_time] = schema[:data_info][:start] if schema.has_key?(:data_info)
attrs[:end_time] = schema[:data_info][:end] attrs[:start_time] = schema[:data_info][:start]
attrs[:total_rows] = schema[:data_info][:rows] attrs[:end_time] = schema[:data_info][:end]
attrs[:total_time] = schema[:data_info][:total_time] attrs[:total_rows] = schema[:data_info][:rows]
attrs[:size_on_disk] = schema[:data_info][:bytes] attrs[:total_time] = schema[:data_info][:total_time]
attrs[:size_on_disk] = schema[:data_info][:bytes]
db_stream.update(attrs) end
db_stream.update!(attrs)
#db_stream.db_elements.destroy_all #db_stream.db_elements.destroy_all
schema[:elements].each do |element_config| schema[:elements].each do |element_config|
element = db_stream.db_elements.find_by_column(element_config[:index]) element = db_stream.db_elements.find_by_column(element_config[:index])
...@@ -146,9 +195,11 @@ module Joule ...@@ -146,9 +195,11 @@ module Joule
# add in extra attributes that require conversion # add in extra attributes that require conversion
attrs[:path] = "#{parent_path}/#{schema[:name]}" attrs[:path] = "#{parent_path}/#{schema[:name]}"
attrs[:joule_id] = schema[:id] attrs[:joule_id] = schema[:id]
attrs[:start_time] = schema[:data_info][:start] if schema.has_key?(:data_info)
attrs[:end_time] = schema[:data_info][:end] attrs[:start_time] = schema[:data_info][:start]
attrs[:event_count] = schema[:data_info][:event_count] attrs[:end_time] = schema[:data_info][:end]
attrs[:event_count] = schema[:data_info][:event_count]
end
attrs[:event_fields_json] = schema[:event_fields].to_json attrs[:event_fields_json] = schema[:event_fields].to_json
event_stream.update(attrs) event_stream.update(attrs)
end end
......
bin/rails
#!/usr/bin/env ruby #!/usr/bin/env ruby
load File.expand_path("spring", __dir__)
APP_PATH = File.expand_path("../config/application", __dir__) APP_PATH = File.expand_path("../config/application", __dir__)
require_relative "../config/boot" require_relative "../config/boot"
require "rails/commands" require "rails/commands"
bin/rake
#!/usr/bin/env ruby #!/usr/bin/env ruby
load File.expand_path("spring", __dir__)
require_relative "../config/boot" require_relative "../config/boot"
require "rake" require "rake"
Rake.application.run Rake.application.run
bin/rspec
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
# The application 'rspec' is installed as part of a gem, and # The application 'rspec' is installed as part of a gem, and
# this file is here to facilitate running it. # this file is here to facilitate running it.
# #
load File.expand_path("spring", __dir__)
require "pathname" require "pathname"
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
......
bin/spring 0 → 100755
#!/usr/bin/env ruby
# This file loads Spring without using loading other gems in the Gemfile, in order to be fast.
# It gets overwritten when you run the `spring binstub` command.
if !defined?(Spring) && [nil, "development", "test"].include?(ENV["RAILS_ENV"])
require "bundler"
Bundler.locked_gems.specs.find { |spec| spec.name == "spring" }&.tap do |spring|
Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path
gem "spring", spring.version
require "spring/binstub"
end
end
config/environments/test.rb
...@@ -9,7 +9,9 @@ Rails.application.configure do ...@@ -9,7 +9,9 @@ Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb. # Settings specified here will take precedence over those in config/application.rb.
# Turn false under Spring and add config.action_view.cache_template_loading = true. # Turn false under Spring and add config.action_view.cache_template_loading = true.
config.cache_classes = true # using spring so following these instructions
config.cache_classes = false
config.action_view.cache_template_loading = true
# Eager loading loads your whole application. When running a single test locally, # Eager loading loads your whole application. When running a single test locally,
# this probably isn't necessary. It's a good idea to do in a continuous integration # this probably isn't necessary. It's a good idea to do in a continuous integration
......
db/migrate/20230623142722_add_timestamps_to_folders_and_streams.rb 0 → 100644
class AddTimestampsToFoldersAndStreams < ActiveRecord::Migration[7.0]
def change
add_column :db_folders, :last_update, :timestamp, default: DateTime.new(1970,1,1,0,0,0)
add_column :db_streams, :last_update, :timestamp, default: DateTime.new(1970,1,1,0,0,0)
add_column :event_streams, :last_update, :timestamp, default: DateTime.new(1970,1,1,0,0,0)
end
end
db/schema.rb
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do ActiveRecord::Schema[7.0].define(version: 2023_06_23_142722) do
create_table "data_apps", force: :cascade do |t| create_table "data_apps", force: :cascade do |t|
t.string "name" t.string "name"
t.string "joule_id" t.string "joule_id"
...@@ -79,6 +79,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do ...@@ -79,6 +79,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do
t.integer "size_on_disk", limit: 8 t.integer "size_on_disk", limit: 8
t.integer "joule_id" t.integer "joule_id"
t.boolean "locked" t.boolean "locked"
t.datetime "last_update", default: "1970-01-01 00:00:00"
t.index ["joule_id"], name: "index_db_folders_on_joule_id" t.index ["joule_id"], name: "index_db_folders_on_joule_id"
end end
...@@ -101,6 +102,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do ...@@ -101,6 +102,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do
t.integer "db_id" t.integer "db_id"
t.integer "joule_id" t.integer "joule_id"
t.boolean "locked" t.boolean "locked"
t.datetime "last_update", default: "1970-01-01 00:00:00"
t.index ["joule_id"], name: "index_db_streams_on_joule_id" t.index ["joule_id"], name: "index_db_streams_on_joule_id"
end end
...@@ -132,6 +134,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do ...@@ -132,6 +134,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_19_150859) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.integer "event_count" t.integer "event_count"
t.string "event_fields_json" t.string "event_fields_json"
t.datetime "last_update", default: "1970-01-01 00:00:00"
t.index ["db_folder_id"], name: "index_event_streams_on_db_folder_id" t.index ["db_folder_id"], name: "index_event_streams_on_db_folder_id"
t.index ["db_id"], name: "index_event_streams_on_db_id" t.index ["db_id"], name: "index_event_streams_on_db_id"
t.index ["joule_id"], name: "index_event_streams_on_joule_id" t.index ["joule_id"], name: "index_event_streams_on_joule_id"
......
spec/adapters/joule/test_schema/0_original_schema.json 0 → 100644
{
"id": 1,
"name": "root",
"description": null,
"locked": true,
"updated_at": "2023-07-01T01:00:46.825436",
"children": [
{
"id": 214,
"name": "folder_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.712913",
"children": [],
"streams": [
{
"id": 210,
"name": "stream_1_1",
"description": "",
"datatype": "float32",
"layout": "float32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.648129",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 569,
"index": 0,
"name": "x",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": 100.0,
"default_min": null
},
{
"id": 570,
"index": 1,
"name": "y",
"units": "",
"plottable": true,
"display_type": "EVENT",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": -6.0
},
{
"id": 571,
"index": 2,
"name": "z",
"units": "watts",
"plottable": true,
"display_type": "DISCRETE",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
},
{
"id": 211,
"name": "stream_1_2",
"description": "",
"datatype": "uint8",
"layout": "uint8_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.712913",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 572,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 573,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 574,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 216,
"name": "folder_3",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.789432",
"children": [
{
"id": 217,
"name": "folder_3_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.789432",
"children": [],
"streams": [
{
"id": 214,
"name": "stream_3_1_1",
"description": "",
"datatype": "int32",
"layout": "int32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.789432",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 580,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 581,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 582,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
}
],
"streams": [
{
"id": 213,
"name": "stream_3_1",
"description": "",
"datatype": "uint16",
"layout": "uint16_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.781562",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 577,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 578,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 579,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 215,
"name": "folder_2",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.806925",
"children": [],
"streams": [
{
"id": 212,
"name": "stream_2_1",
"description": "",
"datatype": "int16",
"layout": "int16_2",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.770594",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 575,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 576,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": [
{
"id": 19,
"name": "transients",
"description": "",
"event_fields": {},
"updated_at": "2023-07-01T01:00:46.797384"
},
{
"id": 20,
"name": "loads",
"description": "",
"event_fields": {},
"updated_at": "2023-07-01T01:00:46.806925"
}
]
},
{
"id": 218,
"name": "folder_4",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.825436",
"children": [
{
"id": 219,
"name": "folder_4_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.825436",
"children": [],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
\ No newline at end of file
spec/adapters/joule/test_schema/1_updated_schema.json 0 → 100644
{
"id": 1,
"name": "root",
"description": null,
"locked": true,
"updated_at": "2023-07-01T01:01:52.380593",
"children": [
{
"id": 218,
"name": "folder_4",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.825436",
"children": [
{
"id": 219,
"name": "folder_4_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:00:46.825436",
"children": [],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
},
{
"id": 216,
"name": "folder_3",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.331388",
"children": [
{
"id": 217,
"name": "folder_3_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.331388",
"children": [],
"streams": [
{
"id": 214,
"name": "stream_3_1_1",
"description": "",
"datatype": "int32",
"layout": "int32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:01:52.331388",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 580,
"index": 0,
"name": "0",
"units": "updated_units",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 581,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 582,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
}
],
"streams": [
{
"id": 213,
"name": "stream_3_1",
"description": "",
"datatype": "uint16",
"layout": "uint16_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.781562",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 577,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 578,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 579,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 214,
"name": "folder_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.359634",
"children": [],
"streams": [
{
"id": 210,
"name": "stream_1_1",
"description": "",
"datatype": "float32",
"layout": "float32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.648129",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 569,
"index": 0,
"name": "x",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": 100.0,
"default_min": null
},
{
"id": 570,
"index": 1,
"name": "y",
"units": "",
"plottable": true,
"display_type": "EVENT",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": -6.0
},
{
"id": 571,
"index": 2,
"name": "z",
"units": "watts",
"plottable": true,
"display_type": "DISCRETE",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
},
{
"id": 211,
"name": "stream_1_2",
"description": "updated_description",
"datatype": "uint8",
"layout": "uint8_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:01:52.359634",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 572,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 573,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 574,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 215,
"name": "folder_2",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.380593",
"children": [],
"streams": [
{
"id": 212,
"name": "stream_2_1",
"description": "",
"datatype": "int16",
"layout": "int16_2",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.770594",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 575,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 576,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": [
{
"id": 20,
"name": "loads",
"description": "",
"event_fields": {},
"updated_at": "2023-07-01T01:00:46.806925"
},
{
"id": 19,
"name": "transients",
"description": "",
"event_fields": {
"updated": "string"
},
"updated_at": "2023-07-01T01:01:52.380593"
}
]
}
],
"streams": [],
"event_streams": []
}
\ No newline at end of file
spec/adapters/joule/test_schema/3_deleted_schema.json 0 → 100644
{
"id": 1,
"name": "root",
"description": null,
"locked": true,
"updated_at": "2023-07-01T01:04:37.485771",
"children": [
{
"id": 214,
"name": "folder_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:06.201445",
"children": [],
"streams": [
{
"id": 210,
"name": "stream_1_1",
"description": "",
"datatype": "float32",
"layout": "float32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.648129",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 569,
"index": 0,
"name": "x",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": 100.0,
"default_min": null
},
{
"id": 570,
"index": 1,
"name": "y",
"units": "",
"plottable": true,
"display_type": "EVENT",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": -6.0
},
{
"id": 571,
"index": 2,
"name": "z",
"units": "watts",
"plottable": true,
"display_type": "DISCRETE",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 218,
"name": "folder_4",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.485771",
"children": [
{
"id": 219,
"name": "folder_4_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.435366",
"children": [],
"streams": [
{
"id": 211,
"name": "stream_1_2",
"description": "updated_description",
"datatype": "uint8",
"layout": "uint8_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:04:06.201445",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 572,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 573,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 574,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": [
{
"id": 20,
"name": "loads",
"description": "",
"event_fields": {},
"updated_at": "2023-07-01T01:00:46.806925"
}
]
},
{
"id": 216,
"name": "folder_3",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.485771",
"children": [
{
"id": 217,
"name": "folder_3_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.331388",
"children": [],
"streams": [
{
"id": 214,
"name": "stream_3_1_1",
"description": "",
"datatype": "int32",
"layout": "int32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:01:52.331388",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 580,
"index": 0,
"name": "0",
"units": "updated_units",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 581,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 582,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
\ No newline at end of file
spec/adapters/joule/test_schema/4_added_schema.json 0 → 100644
{
"id": 1,
"name": "root",
"description": null,
"locked": true,
"updated_at": "2023-07-01T01:04:54.865609",
"children": [
{
"id": 214,
"name": "folder_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:06.201445",
"children": [],
"streams": [
{
"id": 210,
"name": "stream_1_1",
"description": "",
"datatype": "float32",
"layout": "float32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:00:46.648129",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 569,
"index": 0,
"name": "x",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": 100.0,
"default_min": null
},
{
"id": 570,
"index": 1,
"name": "y",
"units": "",
"plottable": true,
"display_type": "EVENT",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": -6.0
},
{
"id": 571,
"index": 2,
"name": "z",
"units": "watts",
"plottable": true,
"display_type": "DISCRETE",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
},
{
"id": 218,
"name": "folder_4",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.485771",
"children": [
{
"id": 219,
"name": "folder_4_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.435366",
"children": [],
"streams": [
{
"id": 211,
"name": "stream_1_2",
"description": "updated_description",
"datatype": "uint8",
"layout": "uint8_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:04:06.201445",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 572,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 573,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 574,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": [
{
"id": 20,
"name": "loads",
"description": "",
"event_fields": {},
"updated_at": "2023-07-01T01:00:46.806925"
}
]
},
{
"id": 216,
"name": "folder_3",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:37.485771",
"children": [
{
"id": 217,
"name": "folder_3_1",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:01:52.331388",
"children": [],
"streams": [
{
"id": 214,
"name": "stream_3_1_1",
"description": "",
"datatype": "int32",
"layout": "int32_3",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:01:52.331388",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 580,
"index": 0,
"name": "0",
"units": "updated_units",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 581,
"index": 1,
"name": "1",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
},
{
"id": 582,
"index": 2,
"name": "2",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": []
}
],
"streams": [],
"event_streams": []
}
],
"streams": [],
"event_streams": []
},
{
"id": 220,
"name": "new",
"description": null,
"locked": false,
"updated_at": "2023-07-01T01:04:54.865609",
"children": [],
"streams": [
{
"id": 216,
"name": "new_data_stream",
"description": "",
"datatype": "float32",
"layout": "float32_1",
"keep_us": -1,
"is_configured": false,
"is_source": false,
"is_destination": false,
"updated_at": "2023-07-01T01:04:54.865609",
"locked": false,
"active": false,
"decimate": true,
"elements": [
{
"id": 586,
"index": 0,
"name": "0",
"units": "",
"plottable": true,
"display_type": "CONTINUOUS",
"offset": 0.0,
"scale_factor": 1.0,
"default_max": null,
"default_min": null
}
]
}
],
"event_streams": [
{
"id": 21,
"name": "new_event_stream",
"description": "",
"event_fields": {
"test": "string"
},
"updated_at": "2023-07-01T01:04:54.851513"
}
]
}
],
"streams": [],
"event_streams": []
}
\ No newline at end of file
spec/adapters/joule/update_db_spec.rb
...@@ -19,19 +19,22 @@ require 'json' ...@@ -19,19 +19,22 @@ require 'json'
# └── folder_4 # └── folder_4
# └── folder_4_1 # └── folder_4_1
def load_schema(schema_name)
raw = File.read(File.dirname(__FILE__)+"/test_schema/#{schema_name}.json")
JSON.parse(raw).deep_symbolize_keys
end
describe Joule::UpdateDb do describe Joule::UpdateDb do
before do before do
raw = File.read(File.dirname(__FILE__)+"/test_db_schema.json") nilm = create(:nilm)
@schema = JSON.parse(raw).deep_symbolize_keys @db = nilm.db
@db = Db.new
end end
let(:dbinfo) { {} } let(:dbinfo) { {} }
describe '*run*' do describe '*run*' do
describe 'given the test database schema' do describe 'given the original schema' do
it 'builds the database' do it 'builds the database' do
service = Joule::UpdateDb.new(@db) service = Joule::UpdateDb.new(@db)
service.run({}, @schema) service.run({}, load_schema('0_original_schema'))
expect(@db.root_folder.subfolders.count).to eq 4 expect(@db.root_folder.subfolders.count).to eq 4
# go through Folder 1 carefully # go through Folder 1 carefully
folder_1 = @db.root_folder.subfolders.where(name: 'folder_1').first folder_1 = @db.root_folder.subfolders.where(name: 'folder_1').first
...@@ -57,15 +60,106 @@ describe Joule::UpdateDb do ...@@ -57,15 +60,106 @@ describe Joule::UpdateDb do
# check for event streams in Folder 2 # check for event streams in Folder 2
folder_2 = @db.root_folder.subfolders.where(name: 'folder_2').first folder_2 = @db.root_folder.subfolders.where(name: 'folder_2').first
expect(folder_2.event_streams.count).to eq 2 expect(folder_2.event_streams.count).to eq 2
events_2_1 = folder_2.event_streams.where(name: 'events_2_1').first load_events = folder_2.event_streams.where(name: 'loads').first
expect(events_2_1.path).to eq '/folder_2/events_2_1' expect(load_events.path).to eq '/folder_2/loads'
expect(events_2_1.joule_id).to eq 100 # aggregate checks
expect(DbElement.count).to eq 14
expect(DbStream.count).to eq 5
expect(DbFolder.count).to eq 7
expect(EventStream.count).to eq 2
#################################
### Now update the schema (1) ###
#################################
puts "##### update schema #######"
folder_4_last_update = DbFolder.where(name:"folder_4").first.updated_at
stream_3_1_last_update = DbStream.where(name:"stream_3_1").first.updated_at
service.run({}, load_schema('1_updated_schema'))
# aggregate checks
expect(@db.root_folder.subfolders.count).to eq 4
expect(DbElement.count).to eq 14
expect(DbStream.count).to eq 5
expect(DbFolder.count).to eq 7
expect(EventStream.count).to eq 2
# stream_3_1_1 has element with new units
folder_3 = @db.root_folder.subfolders.where(name: 'folder_3').first
folder_3_1 = folder_3.subfolders.where(name: 'folder_3_1').first
stream_3_1_1 = folder_3_1.db_streams.where(name: 'stream_3_1_1').first
expect(stream_3_1_1.db_elements.where(name:"0").first.units).to eq "updated_units"
# stream_1_2 has updated description
expect(DbStream.where(name:"stream_1_2").first.description).to eq "updated_description"
# transients event stream has updated event_fields
expect(EventStream.where(name:"transients").count).to eq 1
expect(EventStream.where(name:"transients").first.event_fields).to eq({"updated"=>"string"})
# folder 4 should not be updated
folder_4 = @db.root_folder.subfolders.where(name: 'folder_4').first
expect(folder_4.updated_at).to eq folder_4_last_update
# stream_3_1 should not be updated
expect( DbStream.where(name:"stream_3_1").first.updated_at).to eq stream_3_1_last_update
# quick checks #################################
### Now update the schema (2) ###
#################################
puts "##### move schema #######"
folder_3_orig_id = DbFolder.where(name:"folder_3").first.id
stream_1_2_orig_id = DbStream.find_by_name("stream_1_2").id
service.run({}, load_schema('2_moved_schema'))
# aggregate checks
expect(@db.root_folder.subfolders.count).to eq 3
expect(DbFolder.count).to eq 7
expect(EventStream.count).to eq 2
expect(DbElement.count).to eq 14 expect(DbElement.count).to eq 14
expect(DbStream.count).to eq 5 expect(DbStream.count).to eq 5
# folder 3 has been moved under folder 4
folder_3 = DbFolder.where(name:"folder_3").first
folder_4 = DbFolder.where(name:"folder_4").first
expect(folder_3.parent.id).to eq folder_4.id
expect(folder_3.id).to eq folder_3_orig_id
# stream_1_2 has been moved to folder_4_1
stream_1_2 = DbStream.find_by_name("stream_1_2")
expect(stream_1_2.db_folder.name).to eq "folder_4_1"
expect(stream_1_2.id).to eq stream_1_2_orig_id
#################################
### Now update the schema (3) ###
#################################
puts "##### delete schema #######"
service.run({}, load_schema('3_deleted_schema'))
# aggregate checks
expect(@db.root_folder.subfolders.count).to eq 2
expect(DbFolder.count).to eq 6
expect(EventStream.count).to eq 1
expect(DbElement.count).to eq 9
expect(DbStream.count).to eq 3
# make sure folder_2 and stream_2_1 are gone
expect(DbStream.find_by_name("stream_2_1")).to be nil
expect(DbFolder.find_by_name("folder_2")).to be nil
# make sure stream_3_1 is gone
expect(DbStream.find_by_name("stream_3_1")).to be nil
# loads event stream should be under folder_4_1
expect(EventStream.find_by_name("loads").db_folder.name).to eq "folder_4_1"
#################################
### Now update the schema (4) ###
#################################
puts "##### add schema #######"
service.run({}, load_schema('4_added_schema'))
# aggregate checks
expect(@db.root_folder.subfolders.count).to eq 3
expect(DbFolder.count).to eq 7 expect(DbFolder.count).to eq 7
expect(EventStream.count).to eq 2 expect(EventStream.count).to eq 2
expect(DbElement.count).to eq 10
expect(DbStream.count).to eq 4
# make sure new data and event stream are both present
expect(DbStream.find_by_name("new_data_stream")).not_to be_nil
expect(EventStream.find_by_name("new_event_stream")).not_to be_nil
new_folder = DbFolder.find_by_name("new")
expect(new_folder.db_streams.count).to eq 1
expect(new_folder.event_streams.count).to eq 1
end end
end end
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment