Skip to content

wattsworth / puppet

Go to a project
Commit 4c5ea6a1 by source_reader
Options

Merge branch 'master' of git.wattsworth.net:wattsworth/puppet

parents 75a764a6 ec5afb5b
Expand all
Inline Side-by-side
Showing with 627 additions and 16 deletions
bootstrap.sh
......@@ -12,7 +12,7 @@ EOF
#fi
apt-get update
apt remove --purge libreoffice-* -y > /dev/null
apt-get remove --purge libreoffice-* thunderbird* -y > /dev/null
apt-get upgrade -y
wget https://apt.puppetlabs.com/puppet-release-focal.deb
dpkg -i puppet-release-focal.deb
......
cleanup.sh
#!/bin/bash
set -e
echo "updating all packages and removing unnecessary ones"
apt update
apt-get clean
#apt upgrade -y
#apt autoremove -y
echo "stopping joule and lumen"
service joule stop
nginx -s stop
......@@ -24,7 +30,9 @@ rm -rf /root/puppet
rm -rf /root/.joule
rm -rf ~/.joule
deluser systemd-coredump
echo "If any user shows up below delete with deluser command:"
cat /etc/passwd | grep 999
echo "From cubic remove the ubuntu user with [deluser --remove-home ubuntu]"
history -c
modules/common/manifests/init.pp
......@@ -6,14 +6,14 @@ class common {
# note: chromium for Ubuntu is a snap and cannot be installed in a container
$pkgs=['build-essential','screen','emacs','openvpn',
'nmap','wget', 'sqlite3', 'gparted', 'net-tools',
'openssh-server','zlib1g-dev','chromium-browser',
'openssh-server','zlib1g-dev','curl',
'imagemagick']
package { $pkgs:
ensure => present
}
$pip_pkgs = ['scipy', 'sklearn', 'pandas']
$pip_pkgs = ['scipy', 'sklearn', 'pandas', 'matplotlib']
package { $pip_pkgs:
ensure => present,
provider => pip3
......
modules/docs/files/JUPYTER_README
......@@ -7,6 +7,9 @@ http://localhost:8888).
Create or update the Jupyter password with the following command:
$> sudo jupyter-set-password
To enable access to Joule, run the following command on the Jupyter terminal:
$> sudo -E joule admin authorize
--- (Optional) Access through the Wattsworth Interface ---
To access Jupyter as a Data App add it to the proxy
configuration in [/etc/joule/main.conf]:
......@@ -14,6 +17,7 @@ configuration in [/etc/joule/main.conf]:
For example:
jupyter=http://127.0.0.1:8888
**NOTE: TO USE JUPYTER THROUGH WATTSWORTH (LUMEN)
YOU MUST CHANGE THE NGINX CONFIGURATION
TO SUPPORT SUBDOMAIN APPS**
......
modules/docs/files/STORAGE_README
......@@ -13,7 +13,7 @@ data from persistent instllations.
fields should be 0 and 2 for a second partition or drive.
**If this is the root volume you must enable it on the drive as well**
$> sudo tune2fs -o journal_data_writeback /dev/DEVCIE
$> sudo tune2fs -o journal_data /dev/DEVCIE
-----------------
......
modules/docs/files/nilm/README
......@@ -28,4 +28,6 @@ metric_generator
puppet Scripts for building system images "sticks"
nilm_admin
*To build these projects first extract /opt/asf.tar.gz to src/asf
\ No newline at end of file
*To build these projects first install cross compiler toolchain:
$> sudo apt install gcc-arm-none-eabi and gdb-multiarch
Then extract /opt/asf.tar.gz to src/asf
\ No newline at end of file
modules/docs/manifests/nilm.pp
......@@ -92,9 +92,9 @@ class docs::nilm(String $docs_path, String $owner){
path => ['/bin', '/usr/sbin', '/usr/bin'],
creates => "/opt/asf.tar.gz"
}
package{['gcc-arm-none-eabi','gdb-multiarch']:
ensure => present,
}
#package{['gcc-arm-none-eabi','gdb-multiarch']:
# ensure => present,
#}
vcsrepo{"${src_path}/wemo_firmware":
ensure => latest,
provider => git,
......
modules/joule/files/sudoers
......@@ -6,3 +6,5 @@ joule ALL=(ALL) NOPASSWD:/usr/bin/journalctl -u joule*
joule ALL=(ALL) NOPASSWD:/bin/systemctl status joule
joule ALL=(ALL) NOPASSWD:/bin/systemctl restart joule
joule ALL=(ALL) NOPASSWD:/bin/journalctl -u joule*
joule ALL=(ALL) NOPASSWD:SETENV:/usr/local/bin/joule admin authorize
modules/jupyter/files/API_demo.ipynb 0 → 100644
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Joule API Demonstration Notebook\n",
"\n",
"This notebook shows how to use the Joule Application Programming Interface (API).\n",
"\n",
"\n",
"Before running this notebook, you must authorize API access for the Joule user. To do this run the following command from the terminal:\n",
"```\n",
" $> sudo -E joule admin authorize\n",
" ```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# run this cell first to import the packages\n",
"import joule\n",
"# convenience imports to make code more compact\n",
"from joule.api import Stream, Element, Annotation\n",
"from joule.errors import EmptyPipeError\n",
"import numpy as np\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To use the API you must have access to the Joule node. To view accessible nodes run the following command:\n",
" \n",
" $> joule node list\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# get_node() returns the default node, add a name parameter to request a specific one\n",
"node = joule.api.get_node()\n",
"\n",
"# all node methods are async so you must use the await keyword\n",
"info = await node.info()\n",
"\n",
"print(\"Node [%s] running joule %s\" % (info.name, info.version))\n",
"print(\"%d GiB used\" % (info.size_db / 2**30))\n",
"print(\"%d GiB available\" % (info.size_free / (2**30)))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create streams and write data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# create a two element stream of 5Hz sine, cosine waveforms\n",
"freq = 5.0\n",
"t = np.arange(0,1,0.001) # 1ms sample rate\n",
"sine = np.sin(freq*2*np.pi*t)\n",
"cosine = np.cos(freq*2*np.pi*t)\n",
"tangent = np.tan(freq*2*np.pi*t)\n",
"plt.plot(t, sine, 'r', t, cosine, 'g')\n",
"plt.xlabel('Time (sec)')\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# create a stream on the Joule Node that can store this data\n",
"stream = Stream(name=\"waves\", elements=[Element(name=\"sine\"), Element(name=\"cosine\")])\n",
"stream = await node.stream_create(stream,\"/Demos\") # now stream is a registered model and can be used with API calls\n",
"\n",
"# refresh the node in the Data Explorer and you should see the new stream\n",
"# *NOTE* if you run this code more than once you will receive an error that the stream already exists"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# we need to put the data in an M,3 numpy array:\n",
"# [ ts sine cosine\n",
"# ts sine cosine\n",
"# ... ]\n",
"#\n",
"# There are many ways to do this, the following is rather concise\n",
"# *NOTE* make sure the timestamps are in units of microseconds\n",
"#\n",
"data = np.vstack((t*1e6, sine, cosine)).T\n",
"\n",
"#\n",
"# add data to the stream by using an input pipe\n",
"#\n",
"pipe = await node.data_write(stream)\n",
"await pipe.write(data) # timestamps should be in us\n",
"await pipe.close() # make sure to close the pipe after writing\n",
"\n",
"# refresh the node in the Data Explorer and you should see the new stream\n",
"# *NOTE* if you run this code more than once you will receive an error that the data already exists"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Manipulate streams and data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# get information about a stream\n",
"print(\"Stream Info:\\t\", await node.stream_info(stream))\n",
"\n",
"# get the data intervals (regions of the stream with data)\n",
"print(\"Intervals:\\t\", await node.data_intervals(stream))\n",
"\n",
"# change the display type of an element to discrete\n",
"stream.elements[1].display_type=\"discrete\"\n",
"await node.stream_update(stream) # refresh the node to see this change\n",
"\n",
"# remove data from a stream \n",
"# ***DANGEROUS: OMITTING START and END will remove ALL DATA***\n",
"await node.data_delete(stream,start=0.2*1e6, end=0.4*1e6)\n",
"print(\"--removed data--\")\n",
"print(\"Intervals:\\t\", await node.data_intervals(stream))\n",
"\n",
"# ...many more methods are available, see API docs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Data Annotations"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Annotate a range of data in Lumen then run this cell\n",
"#\n",
"\n",
"# retrieve a list of annotations (include start,end parameters to limit query to a time range)\n",
"annotations = await node.annotation_get(stream)\n",
"\n",
"if len(annotations) == 0:\n",
" print(\"ERROR: Create an annotation in Lumen then run this cell\")\n",
"elif annotations[0].end is None:\n",
" print(\"ERROR: Annotate a range in Lumen, not an event\")\n",
"else:\n",
" annotation = annotations[0]\n",
" \n",
" # read the data associated with the annotation\n",
" pipe = await node.data_read(stream,start=annotation.start, end=annotation.end)\n",
" data = await pipe.read_all() # this automatically closes the pipe\n",
" \n",
" # plot the data\n",
" plt.plot(data['timestamp']/1e6, data['data'])\n",
" plt.title(annotation.title)\n",
" plt.xlabel('Time (sec)')\n",
" plt.show()\n",
"\n",
"# Annotations can also be created with the API\n",
"#\n",
"\n",
" annotation = Annotation(title='API Annotation', start=0.8*1e6)\n",
" await node.annotation_create(annotation, stream)\n",
"\n",
"# refresh the annotations in the Plot Tab of Lumen to see this new annotation\n",
"# *NOTE* If you run this cell multiple times it will create multiple annotations"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Explore Streams and Read Data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Nodes can be explored through the API\n",
"#\n",
"root = await node.folder_root()\n",
"\n",
"def print_folder(folder, indent=0):\n",
" for child in folder.children:\n",
" print(\" \"*indent + child.name)\n",
" print_folder(child, indent+1)\n",
" for stream in folder.streams:\n",
" print(\" \"*indent + \"[%s: %s]\" % (stream.name, stream.layout))\n",
" \n",
"# print the folder directory structure\n",
"print_folder(root)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# streams can be accessed by API object (as shown in previous cells) or by path\n",
"info = await node.stream_info(\"/Demos/waves\")\n",
"print(\"The demo stream has %d rows of data\" % info.rows)\n",
"\n",
"# To read data, first create a pipe to the stream\n",
"# If you want to treat the data like a simple array you can use the read_all method, but if\n",
"# there is too much data this may fail. In general you should treat a pipe as an infinite\n",
"# data source and read it by chunk. This requires more code, but it scales to very large \n",
"# datasets and is the only way to work with realtime data sources\n",
"#\n",
"print(\"-- reading data --\")\n",
"pipe = await node.data_read(\"/Demos/waves\")\n",
"try:\n",
" while True:\n",
" data = await pipe.read()\n",
" plt.plot(data['timestamp']/1e6,data['data'])\n",
" pipe.consume(len(data))\n",
" print(\"%d rows of data\" % len(data))\n",
" # for large data sources the chunk may or may not be an interval boundary\n",
" # you can explicitly check whether this is the end of an interval:\n",
" if pipe.end_of_interval:\n",
" print(\" data boundary\")\n",
"except EmptyPipeError:\n",
" pass\n",
"finally:\n",
" await pipe.close()\n",
"\n",
"plt.xlabel('Time (sec)')\n",
"plt.title('Data showing interval break')\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Reset the Node to original state\n",
"**Run this cell to undo all changes created by this notebook**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"await node.folder_delete(\"/Demos\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.2"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
modules/jupyter/manifests/config.pp
......@@ -7,7 +7,11 @@ class jupyter::config{
owner => joule,
group => joule,
}
file{'/home/joule/API_demo.ipynb':
source => 'puppet:///modules/jupyter/API_demo.ipynb',
owner => joule,
group => joule,
}
file{ '/etc/systemd/system/jupyter.service':
source => 'puppet:///modules/jupyter/jupyter.service',
owner => root,
......
modules/timescaledb/manifests/init.pp
......@@ -24,11 +24,14 @@ class timescaledb {
ensure => 'present',
require => [Apt::Source['postgresql'], Apt::Ppa['ppa:timescale/timescaledb-ppa']]
}
class{ 'postgresql::globals':
version => '12',
}
} else { # on the pi use version 11
class{ 'postgresql::globals':
version => '11',
}
}
# timescaledb is compiled manually on the pi
class{ 'postgresql::globals':
version => '12',
}
class{ 'postgresql::server':
}
postgresql::server::config_entry { 'shared_preload_libraries':
......
rpi_cleanup.sh
#!/bin/bash
set -e
apt-get clean
echo "stopping joule and lumen"
service joule stop
......@@ -80,7 +81,5 @@ echo "7] Enabling first_boot service"
systemctl enable first_boot.service
echo "8] Clearing command history"
history -c
echo "ALL DONE! Remember to remove the puppet directory!!"
tests/apl_nilm_test_script.sh 0 → 100755
#!/bin/bash
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
joule admin authorize
joule master add lumen 127.0.0.1
#----Module Configs----
#Demo Reader
cat > /etc/joule/module_configs/demo_reader.conf <<EOF
[Main]
exec_cmd = joule-random-reader
name = Demo Reader
[Arguments]
width = 2
rate = 10
[Inputs]
# a reader has no input streams
[Outputs]
output = /demo/Random Data
EOF
#Demo Filter
cat > /etc/joule/module_configs/demo_filter.conf <<EOF
[Main]
exec_cmd = joule-mean-filter
name = Demo Filter
[Arguments]
window = 11
[Inputs]
input = /demo/Random Data
[Outputs]
output = /demo/smoothed
EOF
#Visualizer
cat > /etc/joule/module_configs/visualizer.conf <<EOF
[Main]
exec_cmd = joule-visualizer-filter
name = User Interface
is_app = yes
[Arguments]
title = Quick Start Data Pipeline
[Inputs]
smoothed = /demo/smoothed
random = /demo/Random Data
EOF
#LabJack
cat > /etc/joule/module_configs/lj.conf <<EOF
[Main]
name = LabJack
exec_cmd = labjack_u3
[Arguments]
#valid values are [0-15]
channels = 0,1,2
#data rate in Hz
rate = 500
[Outputs]
output = /demo/LabJack Data
EOF
#Meter1 Capture and Meter1 Process
#created by nilm configure
#----Stream Configs----
#/demo/random
cat > /etc/joule/stream_configs/random.conf <<EOF
[Main]
name = Random Data
path = /demo
datatype = float32
keep = 1w
[Element1]
name = rand1
[Element2]
name = rand2
EOF
#/demo/smoothed
cat > /etc/joule/stream_configs/smoothed.conf <<EOF
[Main]
name = smoothed
path = /demo
datatype = float32
keep = 1w
[Element1]
name = filtered1
[Element2]
name = filtered2
EOF
#/demo/lj
cat > /etc/joule/stream_configs/lj.conf <<EOF
[Main]
name = LabJack Data
path = /demo
datatype = float32
keep = 1w
# [channels] number of elements
# units are voltages
[Element1]
name = Pot1
units = V
[Element2]
name = Floating
units = V
[Element3]
name = Pot2
units = V
#additional elements...
EOF
# meter configuration
#meters.yml
cat > /opt/configs/meters.yml <<EOF
meter1:
type: contact
path: /meters/contact
enabled: true # set to false to disable this meter
daq_type: labjack
ip_address: 172.31.33.6
phases: 2 # 1 - 3
sensors:
voltage:
sensor_indices: [3, 4] # maps to phase A,B,C
sensor_scales: 0.0919 # built-in constant
sinefit_phase: A # [A,B,C] voltage used by sinefit
nominal_rms_voltage: 120 # used to scale prep to watts
current:
sensor_indices: [0,1] # maps to phase A,B,C
sensor_scales: [0.0010090,0.00050452]
sinefit_rotations: [0,0] # relative to sinefit_phase voltage
filter:
enable: false
streams:
sinefit:
decimate: true # if [false] only the base stream will be saved
keep: 1m # how much data to keep as [amount][units]
iv: # h: hours, d: days, w: weeks
decimate: true # m: months, y: years
keep: 1w # if [false] no data will be saved
prep:
decimate: true
keep: 3w
nshift: 1
goertzel: false
sensor:
keep: false
meter2:
type: noncontact
path: /meters/noncontact # database location for meter streams
enabled: true # set to false to disable this meter
serial_number: meter2000 # found on the meter case
phases: 2 # 1 - 3
sensors:
voltage:
sensor_index: 7 # electric field sensor
digitally_integrate: false # if true, integrate using FIR filter
nominal_rms_voltage: 120 # used to scale the electric field
current: # --uncomment a line below, or customize--
sensor_indices: [0,1,2,3] # D-Board with 4 A-Boards
calibration:
duration: 9 # length of calibration in seconds
on_duration: 1
off_duration: 2
watts: 200 # power consumed by calibration load
has_neutral: true # [false] if the system has no neutral bus
streams:
sinefit:
decimate: true # if [false] only the base stream will be saved
keep: 1m # how much data to keep as [amount][units]
iv: # h: hours, d: days, w: weeks,
decimate: true # m: months, y: years
keep: 1w # if [false] no data will be saved
prep:
decimate: true
keep: 3w
nshift: 1
goertzel: false
sensor:
keep: false
EOF
#meter2.yml (calibration)
cat > /opt/configs/meters/meter2.yml <<EOF
current_matrix:
- - -0.00012125438257085366
- 0.013679350060842796
- 0.012167743200954548
- 0.00011571253227221393
- - 0.014798241213175594
- 5.03314331965079e-05
- 2.5566000089109125e-05
- 0.021480242055926087
duration: 9
has_neutral: true
off_duration: 2
on_duration: 1
pq_coeffs:
- - - -0.011621992054812813
- 0.003352649533804292
- - 0.999798742714013
- 0.02006175634082956
- - 0.8893134315159313
- 0.018254152891579608
- - 0.004598760118981981
- -0.002041876588625002
- - - 0.6888856214469614
- 0.01093614048128584
- - -0.0008472395183449421
- -0.001612130817487953
- - -0.0016074052596581833
- -0.004407350320724345
- - 0.9999080073416354
- 0.013563806769483603
sensor_matrix:
- - -0.47143977448979235
- 21.75075523821834
- - 40.80884144411953
- -0.02743516239650606
- - 36.29949144228311
- -0.05262827295894099
- - 0.1859606242595248
- 31.569933057364704
sinefit_rotations:
- 0.020063102307395508
- 0.013564222708336394
voltage_scale: 1.0
watts: 200
EOF
nilm configure
#start capture
service joule restart
echo "================================"
echo "Joule started, waiting 5 seconds"
echo "================================"
sleep 5
joule module list
echo "================================"
echo "Demo Reader:"
joule module logs "Demo Reader"
echo "Demo Filter:"
joule module logs "Demo Filter"
echo "LabJack:"
joule module logs "LabJack"
echo "meter1 process:"
joule module logs "meter1 process"
echo "meter1 capture:"
joule module logs "meter1 capture"
echo "meter2 process:"
joule module logs "meter2 process"
echo "meter2 capture:"
joule module logs "meter2 capture"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment