Skip to content

wattsworth / puppet

Go to a project
Commit 4a7e5b22 by source_reader
Options

changed to python virtual environments and apache hosting for nilmdb

parent fd581221
Inline Side-by-side
Showing with 191 additions and 81 deletions
Puppetfile
......@@ -5,3 +5,5 @@ mod 'puppetlabs-concat', '5.2.0'
mod 'puppetlabs-stdlib', '5.1.0'
mod 'puppetlabs-vcsrepo', '3.0.0'
mod 'saz-sudo', '6.0.0'
mod 'puppet-python', '6.2.0'
mod 'puppetlabs-apache', '7.0.0'
modules/common/manifests/init.pp
......@@ -7,14 +7,53 @@ class common {
$pkgs=['build-essential','screen','emacs','openvpn',
'nmap','wget', 'sqlite3', 'gparted', 'net-tools',
'openssh-server','zlib1g-dev','curl',
'imagemagick']
'imagemagick','libfreetype6-dev','python3-tk','libatlas-base-dev']
package { $pkgs:
ensure => present
}
$pip_pkgs = ['scipy', 'sklearn', 'pandas', 'matplotlib']
package { $pip_pkgs:
# configure python virtual environments
file{'/var/venvs':
ensure => directory,
owner => 'root',
group => 'root'
}
python::pyvenv{'joule_venv':
ensure => present,
version => 'system',
systempkgs => false,
venv_dir => '/var/venvs/joule',
require => File['/var/venvs']
}
python::pip{'joule_pkg':
pkgname => 'joule',
virtualenv => '/var/venvs/joule',
require => Python::Pyvenv['joule_venv']
}
if $facts['image_type']=='nilm'{
$nilmdb_pip_pkgs=['cython', 'uwsgi', 'nilmdb','nilmtools']
$nilmdb_pip_url = "https://pypi.wattsworth.net"
python::pyvenv{'nilmdb_venv':
ensure => present,
version => 'system',
systempkgs => false,
venv_dir => '/var/venvs/nilmdb',
require => File['/var/venvs']
}
python::pip{$nilmdb_pip_pkgs:
virtualenv => '/var/venvs/nilmdb',
install_args => "--extra-index-url $nilmdb_pip_url",
require => [Python::Pyvenv['nilmdb_venv']]#,Package[$apt_pkgs]]
}
}
$global_pip_pkgs = ['scipy', 'sklearn', 'pandas', 'matplotlib']
package { $global_pip_pkgs:
ensure => present,
provider => pip3
}
......
modules/joule/manifests/install.pp
......@@ -2,12 +2,15 @@ class joule::install{
require common
require timescaledb
$pip_cmd="pip3 install --upgrade"
exec{'install_joule':
command => "${pip_cmd} joule",
timeout => 0,
path => ['/usr/local/bin','/usr/bin'],
#add symlinks to /usr/local/bin so we can use joule commands
#without activating the venv
file{'/usr/local/bin/joule':
ensure => link,
target => '/var/venvs/joule/bin/joule'
}
file{'/usr/local/bin/jouled':
ensure => link,
target => '/var/venvs/joule/bin/jouled'
}
user{'joule':
......
modules/nginx/files/nilmdb.conf
location /nilmdb {
include uwsgi_params;
uwsgi_buffering off;
uwsgi_pass unix:/tmp/nilmdb.sock;
client_max_body_size 0;
# disable buffer to handle large read requests
uwsgi_max_temp_file_size 0;
proxy_buffering off;
proxy_pass http://localhost:8081;
}
\ No newline at end of file
modules/nilm/manifests/config.pp 0 → 100644
class nilm::config{
$nilm_bin = ['nilm','nilm-reader-capture',
'nilm-filter-prep','nilm-filter-sinefit',
'nilm-filter-reconstructor','nilm-filter-rawtoprep',
'nilm-plug','jim-term']
$nilm_bin.each |String $name| {
file{"/usr/local/bin/$name":
ensure => link,
target => "/var/venvs/joule/bin/$name"
}
}
exec{'initialize_nilm':
command => 'nilm initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/opt/configs/meters.example.yml',
require => File['/usr/local/bin/nilm']
}
exec{'initialize_nilmplug':
command => 'nilm-plug --initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/etc/udev/rules.d/90-nilmplug.rules',
require => File['/usr/local/bin/nilm-plug']
}
}
modules/nilm/manifests/init.pp
class nilm {
$pip_url='https://pypi.wattsworth.net/'
$pip_cmd="pip3 install --upgrade --extra-index-url ${pip_url}"
package{['libfreetype6-dev','python3-tk','libatlas-base-dev']:
ensure => present
}
exec{'install_nilm':
command => "${pip_cmd} nilm",
timeout => 0,
path => ['/usr/local/bin','/usr/bin'],
require => Package['libfreetype6-dev']
}
file{'/opt/configs':
ensure => directory,
#owner => root,
#group => root
}
vcsrepo{'/opt/ethstream':
ensure => latest,
provider => git,
notify => Exec['install_ethstream'],
source => 'https://git.wattsworth.net/wattsworth/ethstream.git',
owner => root
}
exec{'install_ethstream':
command => 'make install',
cwd => '/opt/ethstream',
path => ['/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'],
creates => '/usr/local/bin/ethstream',
require => [Vcsrepo['/opt/ethstream']]
}
exec{'initialize_nilm':
command => 'nilm initialize',
path => ['/usr/local/bin','/usr/sbin','/bin'],
creates => '/opt/configs/meters.example.yml',
require => Exec['install_nilm']
}
require common
contain nilm::install
contain nilm::config
Class['::nilm::install']
-> Class['::nilm::config']
}
modules/nilm/manifests/install.pp 0 → 100644
class nilm::install{
$pip_url='https://pypi.wattsworth.net/'
python::pip{['nilm','nilmplug']:
install_args => "--extra-index-url $pip_url",
virtualenv => '/var/venvs/joule',
require => Python::Pyvenv['joule_venv']
}
file{'/opt/configs':
ensure => directory,
#owner => root,
#group => root
}
vcsrepo{'/opt/ethstream':
ensure => latest,
provider => git,
notify => Exec['install_ethstream'],
source => 'https://git.wattsworth.net/wattsworth/ethstream.git',
owner => root
}
exec{'install_ethstream':
command => 'make install',
cwd => '/opt/ethstream',
path => ['/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'],
creates => '/usr/local/bin/ethstream',
require => [Vcsrepo['/opt/ethstream']]
}
}
modules/nilmdb/files/nilmdb.wsgi
from os.path import exists
import nilmdb.server
#first run nilmdb-fsck if the database exists
if exists("/opt/data/data.sql"):
import nilmdb.fsck
nilmdb.fsck.Fsck("/opt/data",fix=True).check(skip_data=True)
#now start the application
application = nilmdb.server.wsgi_application("/opt/data","/nilmdb")
modules/nilmdb/files/ports.conf 0 → 100644
# Managed by Puppet
Listen 8081
\ No newline at end of file
modules/nilmdb/manifests/config.pp
......@@ -4,20 +4,20 @@ class nilmdb::config{
owner => root,
group => root,
mode => '0644',
notify => Service['nilmdb']
# notify => Service['nilmdb']
}
file{'/etc/nilmdb/nilmdb.wsgi':
owner => root,
group => root,
mode => '0644',
source => 'puppet:///modules/nilmdb/nilmdb.wsgi',
notify => Service['nilmdb']
# notify => Service['nilmdb']
}
file{'/etc/systemd/system/nilmdb.service':
source => "puppet:///modules/nilmdb/nilmdb.service",
owner => root,
group => root,
mode => '0644'
# mode => '0644'
}
}
modules/nilmdb/manifests/init.pp
......@@ -16,10 +16,10 @@ class nilmdb {
contain nilmdb::install
contain nilmdb::config
contain nilmdb::service
# contain nilmdb::service
Class['::nilmdb::install']
-> Class['::nilmdb::config']
~> Class['::nilmdb::service']
# ~> Class['::nilmdb::service']
}
modules/nilmdb/manifests/install.pp
class nilmdb::install{
$apt_pkgs=['python3-dev', 'libpcre3', 'libpcre3-dev']
$pip_pkgs=['cython', 'uwsgi']
package { $apt_pkgs:
ensure => present
}
package { 'python3-psutil': # this should be installed by pip
ensure => absent
# install nilmdb into a virtual environment at /var/venvs/nilmdb
# install apache to run nilmdb with mod_wsgi on port 8081
# exec{'install mod-wsgi-py3':
# command => 'apt install libapache2-mod-wsgi-py3',
# subscribe => [[Class['Apache::Vhost'],Class['Apache']]
# }
class {'apache':
default_vhost => false,
}
package { $pip_pkgs:
provider => pip3,
ensure => present,
require => Package[$apt_pkgs, 'python3-psutil']
class{'apache::mod::wsgi':
wsgi_python_home => '/var/venvs/nilmdb',
package_name => 'libapache2-mod-wsgi-py3',
mod_path => '/usr/lib/apache2/modules/mod_wsgi.so'
}
apache::vhost{'nilmdb-vhost':
use_port_for_filenames => true,
ip => '127.0.0.1',
port => 8081,
docroot => '/nilmdb',
directories => [
{'path'=>'/nilmdb',
'provider'=>'location'}],
wsgi_application_group => '%{GLOBAL}',
wsgi_process_group => 'nilmdb-wsgi',
wsgi_script_aliases => {'/nilmdb'=>'/etc/nilmdb/nilmdb.wsgi'},
wsgi_daemon_process => {'nilmdb-wsgi'=>{
threads => '32',
user => 'nilmdb',
group => 'nilmdb',
}},
require=>User['nilmdb'],
$pip_url='https://pypi.wattsworth.net/'
$pip_cmd="pip3 install --upgrade --extra-index-url ${pip_url}"
exec{'install_nilmdb':
command => "${pip_cmd} nilmdb nilmtools",
timeout => 0,
path => ['/usr/local/bin','/usr/bin'],
require => Package[$pip_pkgs]
}
user{'nilmdb':
ensure=>present,
uid => 902,
managehome=>true
}
#add symlinks to /usr/local/bin so we can use nilmdb commands
#without activating the venv
file{'/usr/local/bin/nilmtool':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmtool'
}
file{'/usr/local/bin/nilmdb-server':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmdb-server'
}
file{'/usr/local/bin/nilmdb-fsck':
ensure => link,
target => '/var/venvs/nilmdb/bin/nilmdb-fsck'
}
file{'/opt/data':
ensure => directory,
owner => joule,
group => joule,
require => User['joule']
owner => nilmdb,
group => nilmdb,
require => User['nilmdb']
}
file{'/etc/nilmdb':
ensure => directory,
owner => root,
......
modules/timescaledb/manifests/init.pp
......@@ -15,7 +15,7 @@ class timescaledb {
repos => 'main',
key => {
'id' => 'B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8',
'server' => 'hkp://p80.pool.sks-keyservers.net:80'
'server' => 'hkps://keyserver.ubuntu.com:443'
}
}
apt::ppa {'ppa:timescale/timescaledb-ppa': }
......
site.pp
......@@ -13,6 +13,5 @@ node 'default' {
if($facts['image_type']=='nilm') {
include nilmdb
include nilm
include nilmplug
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment