Skip to content

wattsworth / puppet

Go to a project
Commit 4a7e5b22 by source_reader
Options

changed to python virtual environments and apache hosting for nilmdb

parent fd581221
Inline Side-by-side
Showing with 191 additions and 81 deletions
Puppetfile
...@@ -5,3 +5,5 @@ mod 'puppetlabs-concat', '5.2.0' ...@@ -5,3 +5,5 @@ mod 'puppetlabs-concat', '5.2.0'
mod 'puppetlabs-stdlib', '5.1.0' mod 'puppetlabs-stdlib', '5.1.0'
mod 'puppetlabs-vcsrepo', '3.0.0' mod 'puppetlabs-vcsrepo', '3.0.0'
mod 'saz-sudo', '6.0.0' mod 'saz-sudo', '6.0.0'
mod 'puppet-python', '6.2.0'
mod 'puppetlabs-apache', '7.0.0'
modules/common/manifests/init.pp
...@@ -7,14 +7,53 @@ class common { ...@@ -7,14 +7,53 @@ class common {
$pkgs=['build-essential','screen','emacs','openvpn', $pkgs=['build-essential','screen','emacs','openvpn',
'nmap','wget', 'sqlite3', 'gparted', 'net-tools', 'nmap','wget', 'sqlite3', 'gparted', 'net-tools',
'openssh-server','zlib1g-dev','curl', 'openssh-server','zlib1g-dev','curl',
'imagemagick'] 'imagemagick','libfreetype6-dev','python3-tk','libatlas-base-dev']
package { $pkgs: package { $pkgs:
ensure => present ensure => present
} }
$pip_pkgs = ['scipy', 'sklearn', 'pandas', 'matplotlib'] # configure python virtual environments
package { $pip_pkgs: file{'/var/venvs':
ensure => directory,
owner => 'root',
group => 'root'
}
python::pyvenv{'joule_venv':
ensure => present,
version => 'system',
systempkgs => false,
venv_dir => '/var/venvs/joule',
require => File['/var/venvs']
}
python::pip{'joule_pkg':
pkgname => 'joule',
virtualenv => '/var/venvs/joule',
require => Python::Pyvenv['joule_venv']
}
if $facts['image_type']=='nilm'{
$nilmdb_pip_pkgs=['cython', 'uwsgi', 'nilmdb','nilmtools']
$nilmdb_pip_url = "https://pypi.wattsworth.net"
python::pyvenv{'nilmdb_venv':
ensure => present,
version => 'system',
systempkgs => false,
venv_dir => '/var/venvs/nilmdb',
require => File['/var/venvs']
}
python::pip{$nilmdb_pip_pkgs:
virtualenv => '/var/venvs/nilmdb',
install_args => "--extra-index-url $nilmdb_pip_url",
require => [Python::Pyvenv['nilmdb_venv']]#,Package[$apt_pkgs]]
}
}
$global_pip_pkgs = ['scipy', 'sklearn', 'pandas', 'matplotlib']
package { $global_pip_pkgs:
ensure => present, ensure => present,
provider => pip3 provider => pip3
} }
......
modules/joule/manifests/install.pp
...@@ -2,12 +2,15 @@ class joule::install{ ...@@ -2,12 +2,15 @@ class joule::install{
require common require common
require timescaledb require timescaledb
$pip_cmd="pip3 install --upgrade" #add symlinks to /usr/local/bin so we can use joule commands
#without activating the venv
exec{'install_joule': file{'/usr/local/bin/joule':
command => "${pip_cmd} joule", ensure => link,
timeout => 0, target => '/var/venvs/joule/bin/joule'
path => ['/usr/local/bin','/usr/bin'], }
file{'/usr/local/bin/jouled':
ensure => link,
target => '/var/venvs/joule/bin/jouled'
} }
user{'joule': user{'joule':
......
modules/nginx/files/nilmdb.conf
location /nilmdb { location /nilmdb {
include uwsgi_params; proxy_buffering off;
uwsgi_buffering off; proxy_pass http://localhost:8081;
uwsgi_pass unix:/tmp/nilmdb.sock;
client_max_body_size 0;
# disable buffer to handle large read requests
uwsgi_max_temp_file_size 0;
} }
\ No newline at end of file
modules/nilm/manifests/config.pp 0 → 100644
class nilm::config{
$nilm_bin = ['nilm','nilm-reader-capture',
'nilm-filter-prep','nilm-filter-sinefit',
'nilm-filter-reconstructor','nilm-filter-rawtoprep',
'nilm-plug','jim-term']
$nilm_bin.each |String $name| {
file{"/usr/local/bin/$name":
ensure => link,
target => "/var/venvs/joule/bin/$name"
}
}
exec{'initialize_nilm':
command => 'nilm initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/opt/configs/meters.example.yml',
require => File['/usr/local/bin/nilm']
}
exec{'initialize_nilmplug':
command => 'nilm-plug --initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/etc/udev/rules.d/90-nilmplug.rules',
require => File['/usr/local/bin/nilm-plug']
}
}
modules/nilm/manifests/init.pp
class nilm { class nilm {
$pip_url='https://pypi.wattsworth.net/' require common
$pip_cmd="pip3 install --upgrade --extra-index-url ${pip_url}" contain nilm::install
contain nilm::config
package{['libfreetype6-dev','python3-tk','libatlas-base-dev']:
ensure => present
}
exec{'install_nilm':
command => "${pip_cmd} nilm",
timeout => 0,
path => ['/usr/local/bin','/usr/bin'],
require => Package['libfreetype6-dev']
}
file{'/opt/configs':
ensure => directory,
#owner => root,
#group => root
}
vcsrepo{'/opt/ethstream':
ensure => latest,
provider => git,
notify => Exec['install_ethstream'],
source => 'https://git.wattsworth.net/wattsworth/ethstream.git',
owner => root
}
exec{'install_ethstream':
command => 'make install',
cwd => '/opt/ethstream',
path => ['/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'],
creates => '/usr/local/bin/ethstream',
require => [Vcsrepo['/opt/ethstream']]
}
exec{'initialize_nilm':
command => 'nilm initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/opt/configs/meters.example.yml',
require => Exec['install_nilm']
}
Class['::nilm::install']
-> Class['::nilm::config']
} }
modules/nilm/manifests/install.pp 0 → 100644
class nilm::install{
$pip_url='https://pypi.wattsworth.net/'
python::pip{['nilm','nilmplug']:
install_args => "--extra-index-url $pip_url",
virtualenv => '/var/venvs/joule',
require => Python::Pyvenv['joule_venv']
}
file{'/opt/configs':
ensure => directory,
#owner => root,
#group => root
}
vcsrepo{'/opt/ethstream':
ensure => latest,
provider => git,
notify => Exec['install_ethstream'],
source => 'https://git.wattsworth.net/wattsworth/ethstream.git',
owner => root
}
exec{'install_ethstream':
command => 'make install',
cwd => '/opt/ethstream',
path => ['/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'],
creates => '/usr/local/bin/ethstream',
require => [Vcsrepo['/opt/ethstream']]
}
}
modules/nilmdb/files/nilmdb.wsgi
from os.path import exists
import nilmdb.server import nilmdb.server
#first run nilmdb-fsck if the database exists
if exists("/opt/data/data.sql"):
import nilmdb.fsck
nilmdb.fsck.Fsck("/opt/data",fix=True).check(skip_data=True)
#now start the application
application = nilmdb.server.wsgi_application("/opt/data","/nilmdb") application = nilmdb.server.wsgi_application("/opt/data","/nilmdb")
modules/nilmdb/files/ports.conf 0 → 100644
# Managed by Puppet
Listen 8081
\ No newline at end of file
modules/nilmdb/manifests/config.pp
...@@ -4,20 +4,20 @@ class nilmdb::config{ ...@@ -4,20 +4,20 @@ class nilmdb::config{
owner => root, owner => root,
group => root, group => root,
mode => '0644', mode => '0644',
notify => Service['nilmdb'] # notify => Service['nilmdb']
} }
file{'/etc/nilmdb/nilmdb.wsgi': file{'/etc/nilmdb/nilmdb.wsgi':
owner => root, owner => root,
group => root, group => root,
mode => '0644', mode => '0644',
source => 'puppet:///modules/nilmdb/nilmdb.wsgi', source => 'puppet:///modules/nilmdb/nilmdb.wsgi',
notify => Service['nilmdb'] # notify => Service['nilmdb']
} }
file{'/etc/systemd/system/nilmdb.service': file{'/etc/systemd/system/nilmdb.service':
source => "puppet:///modules/nilmdb/nilmdb.service", source => "puppet:///modules/nilmdb/nilmdb.service",
owner => root, owner => root,
group => root, group => root,
mode => '0644' # mode => '0644'
} }
} }
modules/nilmdb/manifests/init.pp
...@@ -16,10 +16,10 @@ class nilmdb { ...@@ -16,10 +16,10 @@ class nilmdb {
contain nilmdb::install contain nilmdb::install
contain nilmdb::config contain nilmdb::config
contain nilmdb::service # contain nilmdb::service
Class['::nilmdb::install'] Class['::nilmdb::install']
-> Class['::nilmdb::config'] -> Class['::nilmdb::config']
~> Class['::nilmdb::service'] # ~> Class['::nilmdb::service']
} }
modules/nilmdb/manifests/install.pp
class nilmdb::install{ class nilmdb::install{
$apt_pkgs=['python3-dev', 'libpcre3', 'libpcre3-dev'] # install nilmdb into a virtual environment at /var/venvs/nilmdb
$pip_pkgs=['cython', 'uwsgi']
package { $apt_pkgs:
ensure => present # install apache to run nilmdb with mod_wsgi on port 8081
} # exec{'install mod-wsgi-py3':
package { 'python3-psutil': # this should be installed by pip # command => 'apt install libapache2-mod-wsgi-py3',
ensure => absent # subscribe => [[Class['Apache::Vhost'],Class['Apache']]
# }
class {'apache':
default_vhost => false,
} }
package { $pip_pkgs: class{'apache::mod::wsgi':
provider => pip3, wsgi_python_home => '/var/venvs/nilmdb',
ensure => present, package_name => 'libapache2-mod-wsgi-py3',
require => Package[$apt_pkgs, 'python3-psutil'] mod_path => '/usr/lib/apache2/modules/mod_wsgi.so'
} }
apache::vhost{'nilmdb-vhost':
use_port_for_filenames => true,
ip => '127.0.0.1',
port => 8081,
docroot => '/nilmdb',
directories => [
{'path'=>'/nilmdb',
'provider'=>'location'}],
wsgi_application_group => '%{GLOBAL}',
wsgi_process_group => 'nilmdb-wsgi',
wsgi_script_aliases => {'/nilmdb'=>'/etc/nilmdb/nilmdb.wsgi'},
wsgi_daemon_process => {'nilmdb-wsgi'=>{
threads => '32',
user => 'nilmdb',
group => 'nilmdb',
}},
require=>User['nilmdb'],
$pip_url='https://pypi.wattsworth.net/' }
$pip_cmd="pip3 install --upgrade --extra-index-url ${pip_url}" user{'nilmdb':
ensure=>present,
exec{'install_nilmdb': uid => 902,
command => "${pip_cmd} nilmdb nilmtools", managehome=>true
timeout => 0, }
path => ['/usr/local/bin','/usr/bin'], #add symlinks to /usr/local/bin so we can use nilmdb commands
require => Package[$pip_pkgs] #without activating the venv
file{'/usr/local/bin/nilmtool':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmtool'
}
file{'/usr/local/bin/nilmdb-server':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmdb-server'
}
file{'/usr/local/bin/nilmdb-fsck':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmdb-fsck'
} }
file{'/opt/data': file{'/opt/data':
ensure => directory, ensure => directory,
owner => joule, owner => nilmdb,
group => joule, group => nilmdb,
require => User['joule'] require => User['nilmdb']
} }
file{'/etc/nilmdb': file{'/etc/nilmdb':
ensure => directory, ensure => directory,
owner => root, owner => root,
......
modules/timescaledb/manifests/init.pp
...@@ -15,7 +15,7 @@ class timescaledb { ...@@ -15,7 +15,7 @@ class timescaledb {
repos => 'main', repos => 'main',
key => { key => {
'id' => 'B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8', 'id' => 'B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8',
'server' => 'hkp://p80.pool.sks-keyservers.net:80' 'server' => 'hkps://keyserver.ubuntu.com:443'
} }
} }
apt::ppa {'ppa:timescale/timescaledb-ppa': } apt::ppa {'ppa:timescale/timescaledb-ppa': }
......
site.pp
...@@ -13,6 +13,5 @@ node 'default' { ...@@ -13,6 +13,5 @@ node 'default' {
if($facts['image_type']=='nilm') { if($facts['image_type']=='nilm') {
include nilmdb include nilmdb
include nilm include nilm
include nilmplug
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment