Skip to content

wattsworth / puppet

Go to a project
Commit 0ac90bf1 by source_reader
Options

added test and compressor scripts, fixes for nginx

parent 47c8c2a7
Inline Side-by-side
Showing with 728 additions and 62 deletions
README
......@@ -18,60 +18,3 @@ sudo systemd-nspawn -D wattsworth_cubic/squashfs-root/ --bind /home/jdonnal/pupp
5.) Test with wattsworth test script
==== Raspbian Installation ====
see:
http://blog.oddbit.com/2016/02/07/systemd-nspawn-for-fun-and-well-mostly-f/
1.) extract compressed image
2.)Expand the image to 8G--
qemu-img resize <img> 8G
echo ", +" | sfdisk -N 2 <img>
3.) create a loopback device
(-f: find first unused device, create a partitioned device, show the lo device)
losetup -fP --show <img>
4.) expand the file system
e2fsck -f /dev/loop1p2
resize2fs /dev/loop1p2
3.) mount to rpi/mnt and rpi/mnt/boot
mount /dev/loop1p2 mnt
mount /dev/loop1p1 mnt/boot
4.) copy over the qemu emulator
cp /usr/bin/qemu-arm-static mnt/usr/bin
4.) spawn a container:
systemd-nspawn -D mnt -b --network-interface enxd8eb97bdd8ed --bind /home/jdonnal/puppet:/home/pi/puppet -b --network-interface=enxd8eb97bdd8ed --bind /media/jdonnal/wattsworth_tests:/home/pi/tests -M rpi
#### Install on the Raspberry Pi Directly ####
1.) Update apt sources:
Use mirror at http://mirror.umd.edu/raspbian/raspbian
2.) Increase swap to 2G:
add CONF_SWAPFACTOR=4 to /etc/dphys-swapfile
reboot
free -h
2.) Install Python 3.6
$ wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tar.xz
$ sudo apt-get install build-essential tk-dev libncurses5-dev libncursesw5-dev libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev
tar xf ..
./configure --enable-optimizations
make
make install
3.) Install Numpy and Scipy
sudo apt install libatlas-base-dev
sudo apt build-dep python3 numpy scipy
get the numpy zip file from pypi
4.) Compile Postgresql 10.7 with systemd support and add service file from https://www.postgresql.org/docs/10/server-start.html
5.) Compile timescaledb from source
6.) Install passenger as gem and then run passenger-install-apache2-module (something like that :) )
\ No newline at end of file
apply.sh
......@@ -2,10 +2,14 @@
source set_locale.sh
# install the puppet forge modules
mkdir -p forge_modules
r10k puppetfile install --moduledir=./forge_modules
# remove joule config if present
rm -f /etc/joule/main.conf
# run puppet
export FACTER_PLATFORM="ubuntu"
/opt/puppetlabs/bin/puppet apply --verbose --modulepath=./forge_modules:./modules site.pp
......
cleanup.sh
......@@ -3,7 +3,7 @@ set -e
echo "stopping joule and lumen"
service joule stop
apache2ctl stop
nginx -s stop
echo "1] Remove the databases"
sudo -u postgres psql <<END_OF_SQL
......@@ -20,5 +20,6 @@ echo "5] Enabling first_boot service"
systemctl enable first_boot.service
echo "5] Clearing command history"
history -c
rm -rf /root/puppet
echo "Remember to remove ubuntu user!"
modules/common/files/README
......@@ -3,6 +3,9 @@
To get started:
0.) Optimize postgres+timescale for your hardware by running:
$> sudo timescaledb-tune
1.) Initialize your system and start collecting data:
http://localhost/docs/joule/quick_start.html
-or-
......
modules/common/files/first_boot_gui_notify.sh
#!/bin/bash
pipe=/tmp/first_boot_status
sleep 1
if [[ ! -p $pipe ]]; then
......
modules/common/files/first_boot_status.sh
#!/bin/bash
pipe=/tmp/first_boot_status
if [[ ! -p $pipe ]]; then
......
modules/nginx/files/site.conf
......@@ -38,7 +38,7 @@ server{
passenger_app_root /opt/api;
passenger_document_root /opt/api/public;
passenger_enabled on;
proxy_set_header X-App-Base-Uri "/app";
passenger_set_header X-App-Base-Uri "/app";
}
# Lumen reverse proxy for data apps
location /app {
......
modules/nginx/manifests/ubuntu.pp
class nginx::ubuntu{
notice("including ubuntu nginx")
$deps = ['nginx', 'libnginx-mod-http-passenger']
apt::source { 'passenger':
location => 'https://oss-binaries.phusionpassenger.com/apt/passenger',
release => "${lsbdistcodename}",
repos => 'main',
key => {
'id' => '16378a33a6ef16762922526e561f9b9cac40b2f7', #'561F9B9CAC40B2F7',
'server' => 'hkp://keyserver.ubuntu.com:80'
}
}
package{ $deps:
ensure => present
ensure => present,
require => Apt::Source['passenger']
}
file {'/etc/nginx/nginx.conf':
source => 'puppet:///modules/nginx/nginx.conf',
......@@ -21,6 +34,6 @@ class nginx::ubuntu{
}
service {'nginx':
ensure => running,
require => File['/etc/nginx/site.conf','/etc/nginx/nginx.conf']
require => [File['/etc/nginx/site.conf','/etc/nginx/nginx.conf'], Package['libnginx-mod-http-passenger']]
}
}
rpi_compressor.sh 0 → 100755
#!/bin/bash
set -e
if [ $# -eq 1 ]; then
echo "Copying $1 to rpi.img"
dcfldd if=$1 of=rpi.img
fi
# disk image file
img=rpi.img
#2.) Resize the root partition and store offsets
FS_START=`fdisk -l $img -o start | tail -n 1`
ld=$(losetup --show -f rpi.img -o $(($FS_START * 512)))
e2fsck -f $ld
resize2fs -M $ld
resize2fs -M $ld
resize2fs -M $ld 2> tmp.txt
FS_SIZE=`cat tmp.txt | awk '/blocks/{print}' | grep -o -E '[0-9]+' | head -n 1`
#rm tmp.txt
losetup -d $ld
sleep 0.5
#--debugging--
#cat > vals <<EOF
#FS_START: $FS_START
#FS_SIZE: $FS_SIZE
#EOF
#3.) Update partition table with new parameters
ld=$(losetup --show -f $img)
sfdisk $ld -d > rpi-parts.bak
sfdisk $ld -N2 <<EOF
$FS_START,$(($FS_SIZE * 8))
EOF
#4.) Truncate the image
FS_END=`fdisk -l $img -o end | tail -n 1`
losetup -d $ld
truncate -s $((($FS_END + 1) * 512)) $img
#5.) Zero out extra space
ld=$(losetup --show -f $img -o $(($FS_START * 512)))
mkdir -p /mnt/imageroot
mount $ld /mnt/imageroot
dcfldd if=/dev/zero of=/mnt/imageroot/zero.txt || true
rm /mnt/imageroot/zero.txt
umount /mnt/imageroot
rmdir /mnt/imageroot
losetup -d $ld
#6.) Compress the image
echo "To compress the image run: [zip rpi_img.zip $img]"
tests/nilm_test_script.sh 0 → 100755
#!/bin/bash
#----Module Configs----
#Demo Reader
cat > /etc/joule/module_configs/demo_reader.conf <<EOF
[Main]
exec_cmd = joule-random-reader
name = Demo Reader
[Arguments]
width = 2
rate = 10
[Inputs]
# a reader has no input streams
[Outputs]
output = /demo/Random Data
EOF
#Demo Filter
cat > /etc/joule/module_configs/demo_filter.conf <<EOF
[Main]
exec_cmd = joule-mean-filter
name = Demo Filter
[Arguments]
window = 11
[Inputs]
input = /demo/Random Data
[Outputs]
output = /demo/smoothed
EOF
#Visualizer
cat > /etc/joule/module_configs/visualizer.conf <<EOF
[Main]
exec_cmd = joule-visualizer-filter
name = User Interface
has_interface = yes
[Arguments]
title = Quick Start Data Pipeline
[Inputs]
smoothed = /demo/smoothed
random = /demo/Random Data
EOF
#LabJack
cat > /etc/joule/module_configs/lj.conf <<EOF
[Main]
name = LabJack
exec_cmd = labjack_u3
[Arguments]
#valid values are [0-15]
channels = 0,1,2
#data rate in Hz
rate = 500
[Outputs]
output = /demo/LabJack Data
EOF
#Meter1 Capture and Meter1 Process
#created by nilm configure
#----Stream Configs----
#/demo/random
cat > /etc/joule/stream_configs/random.conf <<EOF
[Main]
name = Random Data
path = /demo
datatype = float32
keep = 1w
[Element1]
name = rand1
[Element2]
name = rand2
EOF
#/demo/smoothed
cat > /etc/joule/stream_configs/smoothed.conf <<EOF
[Main]
name = smoothed
path = /demo
datatype = float32
keep = 1w
[Element1]
name = filtered1
[Element2]
name = filtered2
EOF
#/demo/lj
cat > /etc/joule/stream_configs/lj.conf <<EOF
[Main]
name = LabJack Data
path = /demo
datatype = float32
keep = 1w
# [channels] number of elements
# units are voltages
[Element1]
name = Pot1
units = V
[Element2]
name = Floating
units = V
[Element3]
name = Pot2
units = V
#additional elements...
EOF
# meter configuration
#meters.yml
cat > /opt/configs/meters.yml <<EOF
meter1:
type: contact
path: /meters/contact
enabled: true # set to false to disable this meter
ip_address: 172.31.33.6
phases: 2 # 1 - 3
sensors:
voltage:
sensor_indices: [3,4] # maps to phase A,B,C
sensor_scales: 0.0919 # built-in constant
sinefit_phase: A # [A,B,C] voltage used by sinefit
nominal_rms_voltage: 120 # used to scale prep to watts
current:
sensor_indices: [2,1] # maps to phase A,B,C
sensor_scales: [0.0010090,0.00050452]
sinefit_rotations: [0,0] # relative to sinefit_phase voltage
streams:
sinefit:
decimate: true # if [false] only the base stream will be saved
keep: 1m # how much data to keep as [amount][units]
iv: # h: hours, d: days, w: weeks
decimate: true # m: months, y: years
keep: 1w # if [false] no data will be saved
prep:
decimate: true
keep: 3w
sensor:
keep: false
meter2:
type: noncontact
path: /meters/noncontact # database location for meter streams
enabled: true # set to false to disable this meter
serial_number: meter1008 # found on the meter case
phases: 2 # 1 - 3
sensors:
voltage:
sensor_index: 0 # electric field sensor
digitally_integrate: true # if true, integrate using FIR filter
nominal_rms_voltage: 120 # used to scale the electric field
current: # --uncomment a line below, or customize--
sensor_indices: [1,3,5,7] # D-Board with 4 A-Boards
calibration:
duration: 9 # length of calibration in seconds
watts: 200 # power consumed by calibration load
has_neutral: true # [false] if the system has no neutral bus
streams:
sinefit:
decimate: true # if [false] only the base stream will be saved
keep: 1m # how much data to keep as [amount][units]
iv: # h: hours, d: days, w: weeks,
decimate: true # m: months, y: years
keep: 1w # if [false] no data will be saved
prep:
decimate: true
keep: 3w
sensor:
keep: false
EOF
#meter1.yml (calibration)
cat > /opt/configs/meters/meter2.yml <<EOF
current_matrix:
- - 0.013698286257034089
- -0.008372095473047129
- -7.195016997648725e-05
- 6.943971771534212e-05
- - 0.00045113523026404256
- 0.0010320134824391492
- -0.025266006217889984
- 0.04766767945698939
duration: 9
has_neutral: true
pq_coeffs:
- - - 30791.144718303465
- -1218.828836681789
- - -18821.142653407267
- 755.897624263009
- - -112.74221300378883
- -72.39838665487767
- - 57.49251728329302
- -13.325557613619015
- - - 60.406866609359994
- -31.015634339553483
- - 221.7698659131578
- -29.075136106286408
- - -5026.504793830757
- 214.02621852139075
- - 9484.224738585859
- -380.9201452842851
sensor_matrix:
- - 53.14555615658608
- 0.10624348113745874
- - -32.48605950376758
- 0.3841797932263452
- - -0.1893501464831944
- -8.676794632867567
- - 0.09998591835992797
- 16.370162190401707
sinefit_rotations:
- -0.039563090697980044
- -0.040141975928641264
watts: 200
EOF
nilm configure
#start capture
service joule restart
echo "================================"
echo "Joule started, waiting 5 seconds"
echo "================================"
sleep 5
joule module list
echo "================================"
echo "Demo Reader:"
joule module logs "Demo Reader"
echo "Demo Filter:"
joule module logs "Demo Filter"
echo "LabJack:"
joule module logs "LabJack"
echo "meter1 process:"
joule module logs "meter1 process"
echo "meter1 capture:"
joule module logs "meter1 capture"
echo "meter2 process:"
joule module logs "meter2 process"
echo "meter2 capture:"
joule module logs "meter2 capture"
tests/rpi_nilm_test_script.sh 0 → 100755
#!/bin/bash
#----Module Configs----
#Demo Reader
cat > /etc/joule/module_configs/demo_reader.conf <<EOF
[Main]
exec_cmd = joule-random-reader
name = Demo Reader
[Arguments]
width = 2
rate = 10
[Inputs]
# a reader has no input streams
[Outputs]
output = /demo/Random Data
EOF
#Demo Filter
cat > /etc/joule/module_configs/demo_filter.conf <<EOF
[Main]
exec_cmd = joule-mean-filter
name = Demo Filter
[Arguments]
window = 11
[Inputs]
input = /demo/Random Data
[Outputs]
output = /demo/smoothed
EOF
#Visualizer
cat > /etc/joule/module_configs/visualizer.conf <<EOF
[Main]
exec_cmd = joule-visualizer-filter
name = User Interface
has_interface = yes
[Arguments]
title = Quick Start Data Pipeline
[Inputs]
smoothed = /demo/smoothed
random = /demo/Random Data
EOF
#LabJack
cat > /etc/joule/module_configs/lj.conf <<EOF
[Main]
name = LabJack
exec_cmd = labjack_u3
[Arguments]
#valid values are [0-15]
channels = 0,1,2
#data rate in Hz
rate = 500
[Outputs]
output = /demo/LabJack Data
EOF
#Meter1 Capture and Meter1 Process
#created by nilm configure
#----Stream Configs----
#/demo/random
cat > /etc/joule/stream_configs/random.conf <<EOF
[Main]
name = Random Data
path = /demo
datatype = float32
keep = 1w
[Element1]
name = rand1
[Element2]
name = rand2
EOF
#/demo/smoothed
cat > /etc/joule/stream_configs/smoothed.conf <<EOF
[Main]
name = smoothed
path = /demo
datatype = float32
keep = 1w
[Element1]
name = filtered1
[Element2]
name = filtered2
EOF
#/demo/lj
cat > /etc/joule/stream_configs/lj.conf <<EOF
[Main]
name = LabJack Data
path = /demo
datatype = float32
keep = 1w
# [channels] number of elements
# units are voltages
[Element1]
name = Pot1
units = V
[Element2]
name = Floating
units = V
[Element3]
name = Pot2
units = V
#additional elements...
EOF
# meter configuration
#meters.yml
cat > /opt/configs/meters.yml <<EOF
meter2:
type: noncontact
path: /meters/noncontact # database location for meter streams
enabled: true # set to false to disable this meter
serial_number: meter1008 # found on the meter case
phases: 2 # 1 - 3
sensors:
voltage:
sensor_index: 0 # electric field sensor
digitally_integrate: true # if true, integrate using FIR filter
nominal_rms_voltage: 120 # used to scale the electric field
current: # --uncomment a line below, or customize--
sensor_indices: [1,3,5,7] # D-Board with 4 A-Boards
calibration:
duration: 9 # length of calibration in seconds
watts: 200 # power consumed by calibration load
has_neutral: true # [false] if the system has no neutral bus
streams:
sinefit:
decimate: true # if [false] only the base stream will be saved
keep: false # how much data to keep as [amount][units]
iv: # h: hours, d: days, w: weeks,
decimate: true # m: months, y: years
keep: false # if [false] no data will be saved
prep:
decimate: true
keep: 3w
sensor:
keep: false
EOF
#meter1.yml (calibration)
cat > /opt/configs/meters/meter2.yml <<EOF
current_matrix:
- - 0.013698286257034089
- -0.008372095473047129
- -7.195016997648725e-05
- 6.943971771534212e-05
- - 0.00045113523026404256
- 0.0010320134824391492
- -0.025266006217889984
- 0.04766767945698939
duration: 9
has_neutral: true
pq_coeffs:
- - - 30791.144718303465
- -1218.828836681789
- - -18821.142653407267
- 755.897624263009
- - -112.74221300378883
- -72.39838665487767
- - 57.49251728329302
- -13.325557613619015
- - - 60.406866609359994
- -31.015634339553483
- - 221.7698659131578
- -29.075136106286408
- - -5026.504793830757
- 214.02621852139075
- - 9484.224738585859
- -380.9201452842851
sensor_matrix:
- - 53.14555615658608
- 0.10624348113745874
- - -32.48605950376758
- 0.3841797932263452
- - -0.1893501464831944
- -8.676794632867567
- - 0.09998591835992797
- 16.370162190401707
sinefit_rotations:
- -0.039563090697980044
- -0.040141975928641264
watts: 200
EOF
nilm configure
#start capture
service joule restart
echo "================================"
echo "Joule started, waiting 5 seconds"
echo "================================"
sleep 5
joule module list
echo "================================"
echo "Demo Reader:"
joule module logs "Demo Reader"
echo "Demo Filter:"
joule module logs "Demo Filter"
echo "LabJack:"
joule module logs "LabJack"
echo "meter1 process:"
joule module logs "meter1 process"
echo "meter1 capture:"
joule module logs "meter1 capture"
echo "meter2 process:"
joule module logs "meter2 process"
echo "meter2 capture:"
joule module logs "meter2 capture"
tests/ww_test_script.sh 0 → 100755
#!/bin/bash
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
joule admin authorize
joule master add lumen http://127.0.0.1/lumen
#----Module Configs----
#Demo Reader
cat > /etc/joule/module_configs/data_source.conf <<EOF
[Main]
exec_cmd = joule-random-reader
name = Data Source
[Arguments]
width = 2
rate = 10
[Outputs]
output = /demo/random:float32[x,y]
EOF
#Demo Filter
cat > /etc/joule/module_configs/data_processor.conf <<EOF
[Main]
exec_cmd = joule-mean-filter
name = Data Processor
[Arguments]
window = 11
[Inputs]
input = /demo/random
[Outputs]
output = /demo/smoothed:float32[x,y]
EOF
#Visualizer
cat > /etc/joule/module_configs/data_app.conf <<EOF
[Main]
exec_cmd = joule-visualizer-filter
name = Data App
is_app = yes
[Arguments]
title = Quick Start Data Pipeline
[Inputs]
smoothed = /demo/smoothed
random = /demo/random
EOF
#LabJack
cat > /etc/joule/module_configs/lj.conf <<EOF
[Main]
name = LabJack
exec_cmd = labjack_u3
[Arguments]
#valid values are [0-15]
channels = 0,1,2
#data rate in Hz
rate = 500
[Outputs]
output = /demo/LabJack Data
EOF
#Meter1 Capture and Meter1 Process
#created by nilm configure
#----Stream Configs----
#/demo/random
cat > /etc/joule/stream_configs/random.conf <<EOF
[Main]
name = random
path = /demo
datatype = float32
keep = 1w
[Element1]
name = x
[Element2]
name = y
EOF
#/demo/smoothed
cat > /etc/joule/stream_configs/smoothed.conf <<EOF
[Main]
name = smoothed
path = /demo
datatype = float32
keep = 1w
[Element1]
name = x
[Element2]
name = y
EOF
#/demo/lj
cat > /etc/joule/stream_configs/lj.conf <<EOF
[Main]
name = LabJack Data
path = /demo
datatype = float32
keep = 1w
# [channels] number of elements
# units are voltages
[Element1]
name = Pot1
units = V
[Element2]
name = Floating
units = V
[Element3]
name = Pot2
units = V
#additional elements...
EOF
#start capture
service joule restart
echo "================================"
echo "Joule started, waiting 5 seconds"
echo "================================"
sleep 5
joule module list
echo "================================"
echo "Data Source:"
joule module logs "Data Source"
echo "Data Processor:"
joule module logs "Data Processor"
echo "Data App:"
joule module logs "Data App"
echo "LabJack:"
joule module logs "LabJack"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment