From 452451e8f5c0bd40b867088f481c130a21efb0a1 Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Sun, 4 Nov 2012 21:31:46 +0100 Subject: [PATCH 01/59] python::gunicorn - removed unused/undocumented attributes. --- manifests/gunicorn.pp | 1 - manifests/requirements.pp | 1 - 2 files changed, 2 deletions(-) diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 2548e95d..659555fa 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -46,7 +46,6 @@ $mode = 'wsgi', $dir = false, $bind = false, - $app_interface = 'wsgi', $environment = false, ) { diff --git a/manifests/requirements.pp b/manifests/requirements.pp index b6c9b346..c7b9c255 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -38,7 +38,6 @@ default => "--proxy=${proxy}", } - $req_dir = inline_template('<%= requirements.match(%r!(.+)/.+!)[1] %>') $req_crc = "${requirements}.sha1" file { $requirements: From 5477c34707d6d59b4e7fd0a77c87183b59c37e52 Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Tue, 6 Nov 2012 21:23:46 +0100 Subject: [PATCH 02/59] python::virtualenv - '--system-site-packages' option New 'systempkgs' parameter will call virtualenv with the '--system-site-packages' option, which can greatly speed up creation in some cases. --- README.md | 3 +++ manifests/virtualenv.pp | 14 ++++++++++++-- tests/virtualenv.pp | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7b00d5d..ba4b7668 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,14 @@ Creates Python virtualenv. **proxy** — Proxy server to use for outbound connections. Default: none +**systempkgs** — Copy system site-packages into virtualenv. Default: don't + python::virtualenv { '/var/www/project1': ensure => present, version => 'system', requirements => '/var/www/project1/requirements.txt', proxy => 'http://proxy.domain.com:3128', + systempkgs => true, } ### python::gunicorn diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index f22f4e14..e2866992 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -16,6 +16,9 @@ # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*systempkgs*] +# Copy system site-packages into virtualenv. Default: don't +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -23,6 +26,7 @@ # version => 'system', # requirements => '/var/www/project1/requirements.txt', # proxy => 'http://proxy.domain.com:3128', +# systempkgs => true, # } # # === Authors @@ -33,7 +37,8 @@ $ensure = present, $version = 'system', $requirements = false, - $proxy = false + $proxy = false, + $systempkgs = false, ) { $venv_dir = $name @@ -55,10 +60,15 @@ default => "&& export http_proxy=${proxy}", } + $system_pkgs_flag = $systempkgs ? { + false => '', + default => '--system-site-packages', + } + exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} \ ${proxy_command} \ - && virtualenv -p `which ${python}` ${venv_dir} \ + && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade distribute pip", creates => $venv_dir, } diff --git a/tests/virtualenv.pp b/tests/virtualenv.pp index 190492c7..da5ab57c 100644 --- a/tests/virtualenv.pp +++ b/tests/virtualenv.pp @@ -9,4 +9,5 @@ version => 'system', requirements => '/var/www/project1/requirements.txt', proxy => 'http://proxy.domain.com:3128', + systempkgs => true, } From e63664bc1ccc540a45a1c4eea90a0f7ea482e406 Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Tue, 6 Nov 2012 21:34:03 +0100 Subject: [PATCH 03/59] python::gunicorn - added template parameter The idea is to be able to use a different template, in case the one provided with the module doesn't fit the user's need. --- README.md | 3 +++ manifests/gunicorn.pp | 7 ++++++- tests/gunicorn.pp | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ba4b7668..65cd3912 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ Manages Gunicorn virtual hosts. **environment** — Set ENVIRONMENT variable. Default: none +**template** — Which ERB template to use. Default: python/gunicorn.erb + python::gunicorn { 'vhost': ensure => present, virtualenv => '/var/www/project1', @@ -97,6 +99,7 @@ Manages Gunicorn virtual hosts. dir => '/var/www/project1/current', bind => 'unix:/tmp/gunicorn.socket', environment => 'prod', + template => 'python/gunicorn.erb', } ## Authors diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 659555fa..1b53fc8e 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -25,6 +25,9 @@ # [*environment*] # Set ENVIRONMENT variable. Default: none # +# [*template*] +# Which ERB template to use. Default: python/gunicorn.erb +# # === Examples # # python::gunicorn { 'vhost': @@ -34,6 +37,7 @@ # dir => '/var/www/project1/current', # bind => 'unix:/tmp/gunicorn.socket', # environment => 'prod', +# template => 'python/gunicorn.erb', # } # # === Authors @@ -47,6 +51,7 @@ $dir = false, $bind = false, $environment = false, + $template = 'python/gunicorn.erb', ) { # Parameter validation @@ -59,7 +64,7 @@ mode => '0644', owner => 'root', group => 'root', - content => template('python/gunicorn.erb'), + content => template($template), } } diff --git a/tests/gunicorn.pp b/tests/gunicorn.pp index c081fd64..c32ea1a2 100644 --- a/tests/gunicorn.pp +++ b/tests/gunicorn.pp @@ -11,4 +11,5 @@ dir => '/var/www/project1/current', bind => 'unix:/tmp/gunicorn.socket', environment => 'prod', + template => 'python/gunicorn.erb', } From e7f975a20639d1e3adbfa85f956485238c040506 Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Tue, 6 Nov 2012 21:42:02 +0100 Subject: [PATCH 04/59] updated authors. --- README.md | 2 ++ manifests/gunicorn.pp | 2 ++ manifests/requirements.pp | 1 + manifests/virtualenv.pp | 2 ++ 4 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 65cd3912..272c92a4 100644 --- a/README.md +++ b/README.md @@ -105,3 +105,5 @@ Manages Gunicorn virtual hosts. ## Authors [Sergey Stankevich](https://github.com/stankevich) +[Ashley Penney](https://github.com/apenney) +[Marc Fournier](https://github.com/mfournier) diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 1b53fc8e..13f48720 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -43,6 +43,8 @@ # === Authors # # Sergey Stankevich +# Ashley Penney +# Marc Fournier # define python::gunicorn ( $ensure = present, diff --git a/manifests/requirements.pp b/manifests/requirements.pp index c7b9c255..4dd1f1fe 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -20,6 +20,7 @@ # === Authors # # Sergey Stankevich +# Ashley Penney # define python::requirements ( $virtualenv = 'system', diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index e2866992..e816468f 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -32,6 +32,8 @@ # === Authors # # Sergey Stankevich +# Ashley Penney +# Marc Fournier # define python::virtualenv ( $ensure = present, From 63b4288450a3165cbd15e04fc9480fd16ff53885 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Thu, 8 Nov 2012 07:11:56 -0500 Subject: [PATCH 05/59] Adding pip; modifying Gunicorn template for Django --- README.md | 2 +- manifests/install.pp | 2 +- templates/gunicorn.erb | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7b00d5d..67c9d072 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [puppet-python](https://github.com/stankevich/puppet-python) ====== -Puppet module for installing and managing python, pip, virtualenv, Gunicorn virtual hosts +Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. ## Usage diff --git a/manifests/install.pp b/manifests/install.pp index dec99b8a..e52e7e4c 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -12,7 +12,7 @@ default => absent, } - package { "${python}-dev": ensure => $dev_ensure } + package { [ "${python}-dev", 'python-pip' ]: ensure => $dev_ensure } $venv_ensure = $python::virtualenv ? { true => present, diff --git a/templates/gunicorn.erb b/templates/gunicorn.erb index 2cf3eedd..1a965315 100644 --- a/templates/gunicorn.erb +++ b/templates/gunicorn.erb @@ -30,6 +30,8 @@ CONFIG = { <% end -%> '--workers=<%= @processorcount.to_i*2 %>', '--timeout=30', +<% if mode != 'django' -%> 'app:app', +<% end -%> ), } From 451b21537565b335a45bca0d90ff03567a10e9c2 Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Mon, 11 Mar 2013 14:20:25 +0100 Subject: [PATCH 06/59] Resolve a cyclic dependency There was a dependency between python::virtualenv and python::requirements. Basically, the virtualenv wanted to initialize requirements, but requirements depended on the virtualenv being created already, resulting in: err: Could not apply complete catalog: Found 1 dependency cycle: (Exec[python_requirements_check_/usr/share/err/repo/requirements.txt] => Python::Requirements[/usr/share/err/repo/requirements.txt] => Exec[python_virtualenv_/usr/share/err/python3] => Python::Requirements[/usr/share/err/repo/requirements.txt] => Exec[python_requirements_check_/usr/share/err/repo/requirements.txt]) --- manifests/config.pp | 1 - manifests/virtualenv.pp | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/manifests/config.pp b/manifests/config.pp index 43da22e4..49e52307 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -5,7 +5,6 @@ Class['python::install'] -> Python::Virtualenv <| |> Python::Virtualenv <| |> -> Python::Pip <| |> - Python::Virtualenv <| |> -> Python::Requirements <| |> if $python::gunicorn { Class['python::install'] -> Python::Gunicorn <| |> diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index e816468f..e6c39009 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -76,12 +76,17 @@ } if $requirements { - Exec["python_virtualenv_${venv_dir}"] - -> Python::Requirements[$requirements] + exec { "python_requirements_initial_install_${requirements}": + command => "${venv_dir}/bin/pip install ${proxy_flag} --requirement ${requirements}", + refreshonly => true, + timeout => 1800, + subscribe => Exec["python_virtualenv_${venv_dir}"], + } python::requirements { $requirements: virtualenv => $venv_dir, proxy => $proxy, + require => Exec["python_virtualenv_${venv_dir}"], } } From 191376c9c29a7b069ba618a3b59ecad129d4e320 Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Mon, 11 Mar 2013 14:28:38 +0100 Subject: [PATCH 07/59] Make pip install distribute in a virtualenv optional pip install distribute currentlyfails on Python 3, this way people can skip installing it. It's included by default to remain backward- compatible --- README.md | 3 +++ manifests/virtualenv.pp | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6289fdb0..816610dd 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,15 @@ Creates Python virtualenv. **systempkgs** — Copy system site-packages into virtualenv. Default: don't +**distribute** — Include distribute in the virtualenv. Default: true + python::virtualenv { '/var/www/project1': ensure => present, version => 'system', requirements => '/var/www/project1/requirements.txt', proxy => 'http://proxy.domain.com:3128', systempkgs => true, + distribute => false, } ### python::gunicorn diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index e816468f..c2595e2b 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -19,6 +19,9 @@ # [*systempkgs*] # Copy system site-packages into virtualenv. Default: don't # +# [*distribute*] +# Include distribute in the virtualenv. Default: true +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -41,6 +44,7 @@ $requirements = false, $proxy = false, $systempkgs = false, + $distribute = true, ) { $venv_dir = $name @@ -67,11 +71,16 @@ default => '--system-site-packages', } + $distribute_pkg = $distribute ? { + true => 'distribute', + default => '', + } + exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} \ ${proxy_command} \ && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ - && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade distribute pip", + && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade ${distribute_pkg} pip", creates => $venv_dir, } From 88c178043145581d38c8040baec67051ee5c937a Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Mon, 11 Mar 2013 15:14:08 +0100 Subject: [PATCH 08/59] Allow virtualenv definitions to share the same requirements file --- manifests/requirements.pp | 28 +++++++++++++++++----------- manifests/virtualenv.pp | 11 ++++++----- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 4dd1f1fe..22988e5e 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -4,6 +4,9 @@ # # === Parameters # +# [*requirements*] +# Path to the requirements file. Defaults to the resource name +# # [*virtualenv*] # virtualenv to run pip in. Default: system-wide # @@ -23,12 +26,11 @@ # Ashley Penney # define python::requirements ( - $virtualenv = 'system', - $proxy = false + $requirements = $name, + $virtualenv = 'system', + $proxy = false ) { - $requirements = $name - $pip_env = $virtualenv ? { 'system' => '`which pip`', default => "${virtualenv}/bin/pip", @@ -41,13 +43,17 @@ $req_crc = "${requirements}.sha1" - file { $requirements: - ensure => present, - mode => '0644', - owner => 'root', - group => 'root', - replace => false, - content => '# Puppet will install and/or update pip packages listed here', + # This will ensure multiple python::virtualenv definitions can share the + # the same requirements file. + if !defined(File[$requirements]) { + file { $requirements: + ensure => present, + mode => '0644', + owner => 'root', + group => 'root', + replace => false, + content => '# Puppet will install and/or update pip packages listed here', + } } # SHA1 checksum to detect changes diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index e6c39009..b56da853 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -76,17 +76,18 @@ } if $requirements { - exec { "python_requirements_initial_install_${requirements}": + exec { "python_requirements_initial_install_${requirements}_${venv_dir}": command => "${venv_dir}/bin/pip install ${proxy_flag} --requirement ${requirements}", refreshonly => true, timeout => 1800, subscribe => Exec["python_virtualenv_${venv_dir}"], } - python::requirements { $requirements: - virtualenv => $venv_dir, - proxy => $proxy, - require => Exec["python_virtualenv_${venv_dir}"], + python::requirements { "${requirements}_${venv_dir}": + requirements => $requirements, + virtualenv => $venv_dir, + proxy => $proxy, + require => Exec["python_virtualenv_${venv_dir}"], } } From b1aa3198c8d950b0e0f1254ffc28438a49540297 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sun, 31 Mar 2013 19:36:18 -0400 Subject: [PATCH 09/59] URL support for pip --- README.md | 2 ++ manifests/pip.pp | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 816610dd..aafff6a8 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Installs and manages packages from pip. **virtualenv** — virtualenv to run pip in. +**url** — URL to install from. Default: none + **proxy** — Proxy server to use for outbound connections. Default: none python::pip { 'flask': diff --git a/manifests/pip.pp b/manifests/pip.pp index 9762c103..3a8ec1b2 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -10,6 +10,9 @@ # [*virtualenv*] # virtualenv to run pip in. # +# [*url*] +# URL to install from. Default: none +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # @@ -27,6 +30,7 @@ define python::pip ( $virtualenv, $ensure = present, + $url = false, $proxy = false ) { @@ -45,10 +49,15 @@ default => "^${name}==", } + $source = $url ? { + false => $name, + default => "${url}#egg=${name}", + } + case $ensure { present: { exec { "pip_install_${name}": - command => "${virtualenv}/bin/pip install ${proxy_flag} ${name}", + command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", } } From 7182643bfea6dfa379dbd779b95bfad9aac3f783 Mon Sep 17 00:00:00 2001 From: woldan Date: Mon, 15 Apr 2013 23:17:45 +0200 Subject: [PATCH 10/59] Added `path` to ensure `mkdir` and `virtualenv` are found. --- manifests/virtualenv.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index ace2a00f..b9ab88ed 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -82,6 +82,7 @@ && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade ${distribute_pkg} pip", creates => $venv_dir, + path => [ '/bin', '/usr/bin', '/usr/sbin' ], } if $requirements { From 8b22e3ecd90dd3ac741ff107d6f3fc8c511443ba Mon Sep 17 00:00:00 2001 From: Salimane Adjao Moustapha Date: Fri, 19 Apr 2013 18:11:14 +0800 Subject: [PATCH 11/59] fix python::requirements Error: Parameter cwd failed on Exec and command not found --- manifests/requirements.pp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 22988e5e..cae32bfd 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -31,6 +31,11 @@ $proxy = false ) { + $cwd = $virtualenv ? { + 'system' => '/', + default => "${virtualenv}/bin/pip", + } + $pip_env = $virtualenv ? { 'system' => '`which pip`', default => "${virtualenv}/bin/pip", @@ -46,7 +51,7 @@ # This will ensure multiple python::virtualenv definitions can share the # the same requirements file. if !defined(File[$requirements]) { - file { $requirements: + file { $requirements: ensure => present, mode => '0644', owner => 'root', @@ -64,8 +69,9 @@ } exec { "python_requirements_update_${name}": + provider => shell, command => "${pip_env} install ${proxy_flag} -Ur ${requirements}", - cwd => $virtualenv, + cwd => $cwd, refreshonly => true, timeout => 1800, subscribe => Exec["python_requirements_check_${name}"], From 8e99e56af8b85923aa146d2d0ef4e7b2f92b3471 Mon Sep 17 00:00:00 2001 From: Salimane Adjao Moustapha Date: Fri, 19 Apr 2013 18:26:20 +0800 Subject: [PATCH 12/59] add system virtualenv to tests --- tests/requirements.pp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/requirements.pp b/tests/requirements.pp index a57093f0..9a954fa2 100644 --- a/tests/requirements.pp +++ b/tests/requirements.pp @@ -8,3 +8,8 @@ virtualenv => '/var/www/project1', proxy => 'http://proxy.domain.com:3128', } + +python::requirements { '/var/www/project1/requirements.txt': + virtualenv => 'system', + proxy => 'http://proxy.domain.com:3128', +} From 2712963d1b42b91030a2ccc81c3dcbc3df92ab82 Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Sun, 21 Apr 2013 12:03:43 +0200 Subject: [PATCH 13/59] Conform to Puppet style guide Fixes: puppet-lint "ERROR: two-space soft tabs not used" error --- manifests/requirements.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index cae32bfd..2fabb96a 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -48,10 +48,10 @@ $req_crc = "${requirements}.sha1" - # This will ensure multiple python::virtualenv definitions can share the - # the same requirements file. - if !defined(File[$requirements]) { - file { $requirements: + # This will ensure multiple python::virtualenv definitions can share the + # the same requirements file. + if !defined(File[$requirements]) { + file { $requirements: ensure => present, mode => '0644', owner => 'root', From 156499f5938078b5ab3ec2170f00cd2500f64df1 Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Sun, 21 Apr 2013 12:08:53 +0200 Subject: [PATCH 14/59] Fix regression with $cwd in python::requirements Commit 8b22e3ecd90dd3ac741ff107d6f3fc8c511443ba introduces a regression which in some cases specifies the full path to pip as it's current working directory, rather than the actual directory pip is in. Obviously this will fail. This commit fixes this. --- manifests/requirements.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 2fabb96a..05596325 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -33,7 +33,7 @@ $cwd = $virtualenv ? { 'system' => '/', - default => "${virtualenv}/bin/pip", + default => "${virtualenv}/bin/", } $pip_env = $virtualenv ? { From ee2a67ad8596fa68cb3952e9ee659546f6cbd66d Mon Sep 17 00:00:00 2001 From: Theo Spears Date: Sat, 4 May 2013 17:07:00 +0100 Subject: [PATCH 15/59] Allow setting owner for virtualenv and requirements --- manifests/requirements.pp | 10 +++++++--- manifests/virtualenv.pp | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 05596325..3a32b0ff 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -28,7 +28,9 @@ define python::requirements ( $requirements = $name, $virtualenv = 'system', - $proxy = false + $proxy = false, + $owner = 'root', + $group = 'root' ) { $cwd = $virtualenv ? { @@ -54,8 +56,8 @@ file { $requirements: ensure => present, mode => '0644', - owner => 'root', - group => 'root', + owner => $owner, + group => $group, replace => false, content => '# Puppet will install and/or update pip packages listed here', } @@ -65,6 +67,7 @@ exec { "python_requirements_check_${name}": command => "sha1sum ${requirements} > ${req_crc}", unless => "sha1sum -c ${req_crc}", + user => $owner, require => File[$requirements], } @@ -74,6 +77,7 @@ cwd => $cwd, refreshonly => true, timeout => 1800, + user => $owner, subscribe => Exec["python_requirements_check_${name}"], } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index b9ab88ed..ae0e8472 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -45,6 +45,8 @@ $proxy = false, $systempkgs = false, $distribute = true, + $owner = 'root', + $group = 'root' ) { $venv_dir = $name @@ -81,15 +83,25 @@ ${proxy_command} \ && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade ${distribute_pkg} pip", + user => $owner, creates => $venv_dir, path => [ '/bin', '/usr/bin', '/usr/sbin' ], } + file{$venv_dir: + ensure => directory, + owner => $owner, + group => $group, + recurse => true, + require => Exec["python_virtualenv_${venv_dir}"], + } + if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": command => "${venv_dir}/bin/pip install ${proxy_flag} --requirement ${requirements}", refreshonly => true, timeout => 1800, + user => $owner, subscribe => Exec["python_virtualenv_${venv_dir}"], } @@ -97,6 +109,8 @@ requirements => $requirements, virtualenv => $venv_dir, proxy => $proxy, + owner => $owner, + group => $group, require => Exec["python_virtualenv_${venv_dir}"], } } From 83c9847a506b3f30dedae500f529fc706f96a875 Mon Sep 17 00:00:00 2001 From: Thomas Bartelmess Date: Sun, 12 May 2013 22:26:54 -0400 Subject: [PATCH 16/59] CentOS and RedHat support --- manifests/init.pp | 2 +- manifests/install.pp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index c7ab10f2..8e8de9e2 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -37,7 +37,7 @@ ) { # Module compatibility check - $compatible = [ 'Debian', 'Ubuntu' ] + $compatible = [ 'Debian', 'Ubuntu', 'CentOS', 'RedHat' ] if ! ($::operatingsystem in $compatible) { fail("Module is not compatible with ${::operatingsystem}") } diff --git a/manifests/install.pp b/manifests/install.pp index e52e7e4c..d2acec02 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -5,6 +5,11 @@ default => "python${python::version}", } + $pythondev = $::operatingsystem ? { + 'RedHat', 'CentOS', 'Fedora' => "$python-devel", + 'Debian', 'Ubuntu' => "$python-dev" + } + package { $python: ensure => present } $dev_ensure = $python::dev ? { @@ -12,7 +17,7 @@ default => absent, } - package { [ "${python}-dev", 'python-pip' ]: ensure => $dev_ensure } + package { [ $pythondev, 'python-pip' ]: ensure => $dev_ensure } $venv_ensure = $python::virtualenv ? { true => present, From c1ca0cef3e27e585eca0366c56100f75a60660c1 Mon Sep 17 00:00:00 2001 From: Thomas Bartelmess Date: Sun, 12 May 2013 22:30:41 -0400 Subject: [PATCH 17/59] Use regex for OS matching --- manifests/install.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/install.pp b/manifests/install.pp index d2acec02..cf0a13e0 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -6,8 +6,8 @@ } $pythondev = $::operatingsystem ? { - 'RedHat', 'CentOS', 'Fedora' => "$python-devel", - 'Debian', 'Ubuntu' => "$python-dev" + /(?i:RedHat|CentOS|Fedora)/ => "$python-devel", + /(?i:Debian|Ubuntu)/ => "$python-dev" } package { $python: ensure => present } From 0ee4d8dcbf10145f319920c474053ccb747630af Mon Sep 17 00:00:00 2001 From: Oded Simon Date: Thu, 16 May 2013 12:18:49 +1000 Subject: [PATCH 18/59] add -i
'/var/www/project1/requirements.txt', # proxy => 'http://proxy.domain.com:3128', # systempkgs => true, +# index => 'http://www.example.com/simple/' # } # # === Authors @@ -47,6 +48,7 @@ $distribute = true, $owner = 'root', $group = 'root' + $index = false, ) { $venv_dir = $name @@ -77,12 +79,17 @@ true => 'distribute', default => '', } + $pypi_index = $index ? { + false => '', + default => "-i ${index}", + } + exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} \ ${proxy_command} \ && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ - && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade ${distribute_pkg} pip", + && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, creates => $venv_dir, path => [ '/bin', '/usr/bin', '/usr/sbin' ], @@ -98,7 +105,7 @@ if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": - command => "${venv_dir}/bin/pip install ${proxy_flag} --requirement ${requirements}", + command => "${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --requirement ${requirements}", refreshonly => true, timeout => 1800, user => $owner, From c8693cf024b96d7eacd263cff0dbb48fb2a14b31 Mon Sep 17 00:00:00 2001 From: Aaron DeVore Date: Sat, 18 May 2013 22:45:18 -0700 Subject: [PATCH 19/59] Fix typo from pull request #14 --- manifests/virtualenv.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 9888cd2b..9f9e96c2 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -47,7 +47,7 @@ $systempkgs = false, $distribute = true, $owner = 'root', - $group = 'root' + $group = 'root', $index = false, ) { From 26cb1fb50d87037bc5494124212bdc9165d62b34 Mon Sep 17 00:00:00 2001 From: Aaron DeVore Date: Sat, 18 May 2013 22:52:20 -0700 Subject: [PATCH 20/59] Use the shell in an exec to avoid specifying a path --- manifests/requirements.pp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 3a32b0ff..67d24ce2 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -65,10 +65,11 @@ # SHA1 checksum to detect changes exec { "python_requirements_check_${name}": - command => "sha1sum ${requirements} > ${req_crc}", - unless => "sha1sum -c ${req_crc}", - user => $owner, - require => File[$requirements], + provider => shell, + command => "sha1sum ${requirements} > ${req_crc}", + unless => "sha1sum -c ${req_crc}", + user => $owner, + require => File[$requirements], } exec { "python_requirements_update_${name}": From 5d3b340ac6ebbee8cca959c2653f141e3030f646 Mon Sep 17 00:00:00 2001 From: schacki Date: Mon, 20 May 2013 14:23:14 +0200 Subject: [PATCH 21/59] Removed multiline backslashes from the execution statement, since this can cause issues with vagrant, linux guest, windows host and shared folders --- manifests/virtualenv.pp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 9f9e96c2..fc0f45b7 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -86,10 +86,7 @@ exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} \ - ${proxy_command} \ - && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ - && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, creates => $venv_dir, path => [ '/bin', '/usr/bin', '/usr/sbin' ], From 0132b8fb2abe798f2f27541816bdc71a9289b0ba Mon Sep 17 00:00:00 2001 From: schacki Date: Mon, 20 May 2013 15:17:03 +0200 Subject: [PATCH 22/59] Removed SHA calculation from requirements and applied the audit option the requirements file resource which should do exaxtly the same out of the box --- manifests/requirements.pp | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 67d24ce2..8b94839d 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -48,8 +48,6 @@ default => "--proxy=${proxy}", } - $req_crc = "${requirements}.sha1" - # This will ensure multiple python::virtualenv definitions can share the # the same requirements file. if !defined(File[$requirements]) { @@ -58,28 +56,20 @@ mode => '0644', owner => $owner, group => $group, + audit => content, replace => false, content => '# Puppet will install and/or update pip packages listed here', } } - # SHA1 checksum to detect changes - exec { "python_requirements_check_${name}": - provider => shell, - command => "sha1sum ${requirements} > ${req_crc}", - unless => "sha1sum -c ${req_crc}", - user => $owner, - require => File[$requirements], - } - - exec { "python_requirements_update_${name}": + exec { "python_requirements${name}": provider => shell, - command => "${pip_env} install ${proxy_flag} -Ur ${requirements}", + command => "${pip_env} install ${proxy_flag} -r ${requirements}", cwd => $cwd, refreshonly => true, timeout => 1800, user => $owner, - subscribe => Exec["python_requirements_check_${name}"], + subscribe => File[$requirements], } } From eec58589feaaafd6e3f4e8500df48f5ed833bbdf Mon Sep 17 00:00:00 2001 From: James Carr Date: Thu, 30 May 2013 22:30:15 -0500 Subject: [PATCH 23/59] Added owner and group params for virtualenv. --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index aafff6a8..5a4c8b5b 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,11 @@ Creates Python virtualenv. **distribute** — Include distribute in the virtualenv. Default: true +** owner ** - specify the owner of this virtualenv + +** group ** - specify the group for this virtualenv + + python::virtualenv { '/var/www/project1': ensure => present, version => 'system', @@ -77,6 +82,8 @@ Creates Python virtualenv. proxy => 'http://proxy.domain.com:3128', systempkgs => true, distribute => false, + owner => 'appuser', + group => 'apps', } ### python::gunicorn From a0e9b8c1958966d0d783f1d71b7b5fe3be7ba02a Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 19:38:35 +1000 Subject: [PATCH 24/59] Added braces around python variable to avoid Puppet syntax errors in install manifest --- manifests/install.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/install.pp b/manifests/install.pp index cf0a13e0..6454377b 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -6,8 +6,8 @@ } $pythondev = $::operatingsystem ? { - /(?i:RedHat|CentOS|Fedora)/ => "$python-devel", - /(?i:Debian|Ubuntu)/ => "$python-dev" + /(?i:RedHat|CentOS|Fedora)/ => "${python}-devel", + /(?i:Debian|Ubuntu)/ => "${python}-dev" } package { $python: ensure => present } From 64de5abe35e71ad85b971bcf9f99a0ea738a41e9 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 19:39:41 +1000 Subject: [PATCH 25/59] Specifically looked for the activate script when creating a virtualenv since this is created last. This ensures that a half-created virtualenv will be detected and corrected by Puppet. --- manifests/virtualenv.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index fc0f45b7..9547e270 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -88,7 +88,7 @@ exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, - creates => $venv_dir, + creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], } From 3c606c95e12be941529fdf86c14f3f35bcddbf69 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 19:41:24 +1000 Subject: [PATCH 26/59] Added owner, group to the pip class which ensures that packages will be installed with the correct permissions. --- manifests/pip.pp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index 3a8ec1b2..b40b916a 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -29,9 +29,11 @@ # define python::pip ( $virtualenv, - $ensure = present, - $url = false, - $proxy = false + $ensure = present, + $url = false, + $owner = 'root', + $group = 'root', + $proxy = false ) { # Parameter validation @@ -57,15 +59,17 @@ case $ensure { present: { exec { "pip_install_${name}": - command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", - unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", + unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + user => $owner, } } default: { exec { "pip_uninstall_${name}": - command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", - onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", + onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + user => $owner, } } } From 5617ad20f4515254d9ac7b9cdef63de59ebece08 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 19:43:14 +1000 Subject: [PATCH 27/59] Added environment attribute to the pip class which allows for users to override environment variables like LD_LIBRARY_PATH and ORACLE_HOME which are needed to install packages like cx_Oracle. --- manifests/pip.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index b40b916a..73f4cdc4 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -33,7 +33,8 @@ $url = false, $owner = 'root', $group = 'root', - $proxy = false + $proxy = false, + environment = [] ) { # Parameter validation @@ -62,6 +63,7 @@ command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", user => $owner, + environment => $environment, } } @@ -70,6 +72,7 @@ command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", user => $owner, + environment => $environment, } } } From f6fd9f46fe9c058d1f0877043d42829efe0c45cb Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 20:48:10 +1000 Subject: [PATCH 28/59] Now that all exec statements run under a particular owner, there is no need for the CPU intensive recursive virtualenv permission check --- manifests/virtualenv.pp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 9547e270..5946e157 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -84,7 +84,6 @@ default => "-i ${index}", } - exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, @@ -92,14 +91,6 @@ path => [ '/bin', '/usr/bin', '/usr/sbin' ], } - file{$venv_dir: - ensure => directory, - owner => $owner, - group => $group, - recurse => true, - require => Exec["python_virtualenv_${venv_dir}"], - } - if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": command => "${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --requirement ${requirements}", From 1aaef848f4a52d83a6b44eeb4cffd9b61a665c41 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 23:13:50 +1000 Subject: [PATCH 29/59] Made system virtualenv more robust and ensured that pip logging would occur in an appropriate directory upon failure to avoid errors --- manifests/pip.pp | 24 +++++++++++++++++++----- manifests/requirements.pp | 9 ++++++--- manifests/virtualenv.pp | 4 ++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index 73f4cdc4..0693057b 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -28,7 +28,7 @@ # Sergey Stankevich # define python::pip ( - $virtualenv, + $virtualenv = 'system', $ensure = present, $url = false, $owner = 'root', @@ -42,6 +42,20 @@ fail('python::pip: virtualenv parameter must not be empty') } + if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { + fail('python::pip: root user must be used when virtualenv is system') + } + + $cwd = $virtualenv ? { + 'system' => '/', + default => "${virtualenv}", + } + + $pip_env = $virtualenv ? { + 'system' => '`which pip`', + default => "${virtualenv}/bin/pip", + } + $proxy_flag = $proxy ? { false => '', default => "--proxy=${proxy}", @@ -60,8 +74,8 @@ case $ensure { present: { exec { "pip_install_${name}": - command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", - unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "$pip_env --log-file ${cwd}/pip.log install ${proxy_flag} ${source}", + unless => "$pip_env freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, } @@ -69,8 +83,8 @@ default: { exec { "pip_uninstall_${name}": - command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", - onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "echo y | $pip_env uninstall ${proxy_flag} ${name}", + onlyif => "$pip_env freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 8b94839d..3019a469 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -33,9 +33,13 @@ $group = 'root' ) { + if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { + fail('python::pip: root user must be used when virtualenv is system') + } + $cwd = $virtualenv ? { 'system' => '/', - default => "${virtualenv}/bin/", + default => "${virtualenv}", } $pip_env = $virtualenv ? { @@ -64,8 +68,7 @@ exec { "python_requirements${name}": provider => shell, - command => "${pip_env} install ${proxy_flag} -r ${requirements}", - cwd => $cwd, + command => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} -r ${requirements}", refreshonly => true, timeout => 1800, user => $owner, diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 5946e157..53f30f8a 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -85,7 +85,7 @@ } exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], @@ -93,7 +93,7 @@ if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": - command => "${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --requirement ${requirements}", + command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --requirement ${requirements}", refreshonly => true, timeout => 1800, user => $owner, From 5132cf1798d862e28f9c9355a2e01a96f585f28f Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Mon, 3 Jun 2013 23:29:44 +1000 Subject: [PATCH 30/59] Removed owner attribute from the pip class as it was not needed and touched up a few mistakes. --- manifests/pip.pp | 15 +++++++-------- manifests/virtualenv.pp | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index 0693057b..3940374f 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -28,13 +28,12 @@ # Sergey Stankevich # define python::pip ( - $virtualenv = 'system', - $ensure = present, - $url = false, - $owner = 'root', - $group = 'root', - $proxy = false, - environment = [] + $ensure = present, + $virtualenv = 'system', + $url = false, + $owner = 'root', + $proxy = false, + $environment = [] ) { # Parameter validation @@ -42,7 +41,7 @@ fail('python::pip: virtualenv parameter must not be empty') } - if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { + if $virtualenv == 'system' and $owner != 'root' { fail('python::pip: root user must be used when virtualenv is system') } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 53f30f8a..684a4175 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -93,7 +93,7 @@ if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": - command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --requirement ${requirements}", + command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} -r ${requirements}", refreshonly => true, timeout => 1800, user => $owner, From 01bf2791ecadfbb954b947fefa8b15ca0c0b9947 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 09:36:33 +1000 Subject: [PATCH 31/59] Amended all documentation and updated version to 1.1.0 --- Modulefile | 2 +- README.md | 87 ++++++++++++++++++++++++++------------- manifests/pip.pp | 7 ++++ manifests/requirements.pp | 7 ++++ manifests/virtualenv.pp | 10 +++++ 5 files changed, 84 insertions(+), 29 deletions(-) diff --git a/Modulefile b/Modulefile index c7f2f97f..e132e991 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.0.0' +version '1.1.0' author 'Sergey Stankevich' license 'Apache License, Version 2.0' diff --git a/README.md b/README.md index 5a4c8b5b..1108ba03 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,36 @@ Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. +**Version 1.1.0 Notes** + +Version 1.1.0 makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. + +Please note that everal changes have been made in v1.1.0 which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. + +Currently, the changes you need to make are as follows: + +* All pip definitions MUST include the owner field which specifies which user owns the virtualenv that packages will be installed in. Adding this greatly improves performance and efficiency of this module. + +## Installation + +``` bash +cd /etc/puppet/modules +git clone git://github.com/stankevich/puppet-python.git +``` + ## Usage ### python Installs and manages python, python-dev, python-virtualenv and Gunicorn. -**version** — Python version to install. Default: system default +**version** - Python version to install. Default: system default -**dev** — Install python-dev. Default: false +**dev** - Install python-dev. Default: false -**virtualenv** — Install python-virtualenv. Default: false +**virtualenv** - Install python-virtualenv. Default: false -**gunicorn** — Install Gunicorn. Default: false +**gunicorn** - Install Gunicorn. Default: false class { 'python': version => 'system', @@ -28,52 +45,65 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn. Installs and manages packages from pip. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present -**virtualenv** — virtualenv to run pip in. +**virtualenv** - virtualenv to run pip in. Default: system (no virtualenv) -**url** — URL to install from. Default: none +**url** - URL to install from. Default: none -**proxy** — Proxy server to use for outbound connections. Default: none +**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root - python::pip { 'flask': - virtualenv => '/var/www/project1', - proxy => 'http://proxy.domain.com:3128', +**proxy** - Proxy server to use for outbound connections. Default: none + +**environment** - Additional environment variables required to install the packages. Default: none + + python::pip { 'cx_Oracle': + virtualenv => '/var/www/project1', + owner => 'appuser', + proxy => 'http://proxy.domain.com:3128', + environment => 'ORACLE_HOME=/usr/lib/oracle/11.2/client64', } ### python::requirements Installs and manages Python packages from requirements file. -**virtualenv** — virtualenv to run pip in. Default: system-wide +**virtualenv** - virtualenv to run pip in. Default: system-wide + +**proxy** - Proxy server to use for outbound connections. Default: none + +**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root -**proxy** — Proxy server to use for outbound connections. Default: none +**group** - The group that was used to create the virtualenv. This is used to create the requirements file with correct permissions if it's not present already. python::requirements { '/var/www/project1/requirements.txt': virtualenv => '/var/www/project1', proxy => 'http://proxy.domain.com:3128', + owner => 'appuser', + group => 'apps', } ### python::virtualenv Creates Python virtualenv. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present -**version** — Python version to use. Default: system default +**version** - Python version to use. Default: system default -**requirements** — Path to pip requirements.txt file. Default: none +**requirements** - Path to pip requirements.txt file. Default: none -**proxy** — Proxy server to use for outbound connections. Default: none +**proxy** - Proxy server to use for outbound connections. Default: none -**systempkgs** — Copy system site-packages into virtualenv. Default: don't +**systempkgs** - Copy system site-packages into virtualenv. Default: don't -**distribute** — Include distribute in the virtualenv. Default: true +**distribute** - Include distribute in the virtualenv. Default: true -** owner ** - specify the owner of this virtualenv +**owner** - Specify the owner of this virtualenv -** group ** - specify the group for this virtualenv +**group** - Specify the group for this virtualenv +**index** - Base URL of Python package index. Default: none python::virtualenv { '/var/www/project1': ensure => present, @@ -90,19 +120,19 @@ Creates Python virtualenv. Manages Gunicorn virtual hosts. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present -**virtualenv** — Run in virtualenv, specify directory. Default: disabled +**virtualenv** - Run in virtualenv, specify directory. Default: disabled -**mode** — Gunicorn mode. wsgi/django. Default: wsgi +**mode** - Gunicorn mode. wsgi/django. Default: wsgi -**dir** — Application directory. +**dir** - Application directory. -**bind** — Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket +**bind** - Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket -**environment** — Set ENVIRONMENT variable. Default: none +**environment** - Set ENVIRONMENT variable. Default: none -**template** — Which ERB template to use. Default: python/gunicorn.erb +**template** - Which ERB template to use. Default: python/gunicorn.erb python::gunicorn { 'vhost': ensure => present, @@ -119,3 +149,4 @@ Manages Gunicorn virtual hosts. [Sergey Stankevich](https://github.com/stankevich) [Ashley Penney](https://github.com/apenney) [Marc Fournier](https://github.com/mfournier) +[Fotis Gimian](https://github.com/fgimian) diff --git a/manifests/pip.pp b/manifests/pip.pp index 3940374f..a9805d34 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -13,9 +13,15 @@ # [*url*] # URL to install from. Default: none # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# # === Examples # # python::pip { 'flask': @@ -26,6 +32,7 @@ # === Authors # # Sergey Stankevich +# Fotis Gimian # define python::pip ( $ensure = present, diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 3019a469..789c0d3c 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -13,6 +13,12 @@ # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# # === Examples # # python::requirements { '/var/www/project1/requirements.txt': @@ -24,6 +30,7 @@ # # Sergey Stankevich # Ashley Penney +# Fotis Gimian # define python::requirements ( $requirements = $name, diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 684a4175..f45678be 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -22,6 +22,15 @@ # [*distribute*] # Include distribute in the virtualenv. Default: true # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# +# [*index*] +# Base URL of Python package index. Default: none (http://pypi.python.org/simple/) +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -38,6 +47,7 @@ # Sergey Stankevich # Ashley Penney # Marc Fournier +# Fotis Gimian # define python::virtualenv ( $ensure = present, From 17debdc04506a2bfd4297b7620acffdbd01c9176 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 10:20:38 +1000 Subject: [PATCH 32/59] Made the pip package independent from the dev package in the python class and updated version to 1.1.1 --- Modulefile | 2 +- README.md | 9 ++++++--- manifests/init.pp | 5 +++++ manifests/install.pp | 8 +++++++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Modulefile b/Modulefile index e132e991..fc139205 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.1.0' +version '1.1.1' author 'Sergey Stankevich' license 'Apache License, Version 2.0' diff --git a/README.md b/README.md index 1108ba03..d072b768 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,16 @@ Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. -**Version 1.1.0 Notes** +**Version 1.1.1 Notes** -Version 1.1.0 makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. +Version 1.1.1 makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. -Please note that everal changes have been made in v1.1.0 which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. +Please note that everal changes have been made in v1.1.1 which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. Currently, the changes you need to make are as follows: * All pip definitions MUST include the owner field which specifies which user owns the virtualenv that packages will be installed in. Adding this greatly improves performance and efficiency of this module. +* You must explicitly specify pip => true in the python class if you want pip installed. As such, the pip package is now independent of the dev package and so one can exist without the other. ## Installation @@ -28,6 +29,8 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn. **version** - Python version to install. Default: system default +**pip** - Install python-pip. Default: false + **dev** - Install python-dev. Default: false **virtualenv** - Install python-virtualenv. Default: false diff --git a/manifests/init.pp b/manifests/init.pp index 8e8de9e2..2a9a44c3 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -7,6 +7,9 @@ # [*version*] # Python version to install. Default: system default # +# [*pip*] +# Install python-pip. Default: false +# # [*dev*] # Install python-dev. Default: false # @@ -20,6 +23,7 @@ # # class { 'python': # version => 'system', +# pip => true, # dev => true, # virtualenv => true, # gunicorn => true, @@ -31,6 +35,7 @@ # class python ( $version = 'system', + $pip = false, $dev = false, $virtualenv = false, $gunicorn = false diff --git a/manifests/install.pp b/manifests/install.pp index 6454377b..9306e3a3 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -17,7 +17,13 @@ default => absent, } - package { [ $pythondev, 'python-pip' ]: ensure => $dev_ensure } + $pip_ensure = $python::pip ? { + true => present, + default => absent, + } + + package { $pythondev: ensure => $dev_ensure } + package { 'python-pip': ensure => $pip_ensure } $venv_ensure = $python::virtualenv ? { true => present, From 8608958fe1bb7fec4bb75c65bd6b1f1e236e88bd Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 11:25:48 +1000 Subject: [PATCH 33/59] Removed unnecessary which commands for obtaining the pip and python executables --- manifests/pip.pp | 2 +- manifests/requirements.pp | 2 +- manifests/virtualenv.pp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index a9805d34..ddbcd6fd 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -58,7 +58,7 @@ } $pip_env = $virtualenv ? { - 'system' => '`which pip`', + 'system' => 'pip', default => "${virtualenv}/bin/pip", } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 789c0d3c..ba16d1e2 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -50,7 +50,7 @@ } $pip_env = $virtualenv ? { - 'system' => '`which pip`', + 'system' => 'pip', default => "${virtualenv}/bin/pip", } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index f45678be..7fbf25ef 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -95,7 +95,7 @@ } exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", user => $owner, creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], From 167b43b8b0fb54311c7703a1bbb627a54e84ebab Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 11:26:22 +1000 Subject: [PATCH 34/59] Touched up virtualenv to run from /tmp to avoid write errors --- manifests/virtualenv.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 7fbf25ef..07007fb5 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -99,6 +99,7 @@ user => $owner, creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], + cwd => "/tmp", } if $requirements { From a6d9f59d0bd9c27e1fd9668b926913c44dfb52e6 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 11:26:46 +1000 Subject: [PATCH 35/59] Incremented version to 1.1.2 --- Modulefile | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modulefile b/Modulefile index fc139205..dfd366b7 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.1.1' +version '1.1.2' author 'Sergey Stankevich' license 'Apache License, Version 2.0' diff --git a/README.md b/README.md index d072b768..2cf6b7d0 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. -**Version 1.1.1 Notes** +**Version 1.1.x Notes** -Version 1.1.1 makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. +Version 1.1.x makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. -Please note that everal changes have been made in v1.1.1 which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. +Please note that everal changes have been made in v1.1.x which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. Currently, the changes you need to make are as follows: From d3d69488a9a9f2bf2edd170ea6db31830690e8d9 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 11:55:04 +1000 Subject: [PATCH 36/59] Added environment parameter to requirements and virtualenv classes and incremented version to 1.1.3 --- Modulefile | 2 +- manifests/requirements.pp | 15 ++++++++++----- manifests/virtualenv.pp | 20 +++++++++++++------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Modulefile b/Modulefile index dfd366b7..1fb0c2da 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.1.2' +version '1.1.3' author 'Sergey Stankevich' license 'Apache License, Version 2.0' diff --git a/manifests/requirements.pp b/manifests/requirements.pp index ba16d1e2..67906fc8 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -10,15 +10,18 @@ # [*virtualenv*] # virtualenv to run pip in. Default: system-wide # -# [*proxy*] -# Proxy server to use for outbound connections. Default: none -# # [*owner*] # The owner of the virtualenv being manipulated. Default: root # # [*group*] # The group relating to the virtualenv being manipulated. Default: root # +# [*proxy*] +# Proxy server to use for outbound connections. Default: none +# +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# # === Examples # # python::requirements { '/var/www/project1/requirements.txt': @@ -35,9 +38,10 @@ define python::requirements ( $requirements = $name, $virtualenv = 'system', - $proxy = false, $owner = 'root', - $group = 'root' + $group = 'root', + $proxy = false, + $environment = [] ) { if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { @@ -80,6 +84,7 @@ timeout => 1800, user => $owner, subscribe => File[$requirements], + environment => $environment, } } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 07007fb5..354e4d82 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -13,23 +13,26 @@ # [*requirements*] # Path to pip requirements.txt file. Default: none # -# [*proxy*] -# Proxy server to use for outbound connections. Default: none -# # [*systempkgs*] # Copy system site-packages into virtualenv. Default: don't # # [*distribute*] # Include distribute in the virtualenv. Default: true # +# [*index*] +# Base URL of Python package index. Default: none (http://pypi.python.org/simple/) +# # [*owner*] # The owner of the virtualenv being manipulated. Default: root # # [*group*] # The group relating to the virtualenv being manipulated. Default: root # -# [*index*] -# Base URL of Python package index. Default: none (http://pypi.python.org/simple/) +# [*proxy*] +# Proxy server to use for outbound connections. Default: none +# +# [*environment*] +# Additional environment variables required to install the packages. Default: none # # === Examples # @@ -53,12 +56,13 @@ $ensure = present, $version = 'system', $requirements = false, - $proxy = false, $systempkgs = false, $distribute = true, + $index = false, $owner = 'root', $group = 'root', - $index = false, + $proxy = false, + $environment = [] ) { $venv_dir = $name @@ -100,6 +104,7 @@ creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], cwd => "/tmp", + environment => $environment, } if $requirements { @@ -109,6 +114,7 @@ timeout => 1800, user => $owner, subscribe => Exec["python_virtualenv_${venv_dir}"], + environment => $environment, } python::requirements { "${requirements}_${venv_dir}": From 96e50bf5038add993258f6d8f4b317cf3c779d33 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 12:04:18 +1000 Subject: [PATCH 37/59] Fixed minor typo in installation instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cf6b7d0..971bbe87 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Currently, the changes you need to make are as follows: ``` bash cd /etc/puppet/modules -git clone git://github.com/stankevich/puppet-python.git +git clone git://github.com/stankevich/puppet-python.git python ``` ## Usage From 07e6cc6b3d2afa25be0ae829c5bff8385c2815f3 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Tue, 4 Jun 2013 12:15:01 +1000 Subject: [PATCH 38/59] Changed documentation to reflect that it's a fork and that I will be maintaining it --- Modulefile | 8 ++++---- README.md | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Modulefile b/Modulefile index 1fb0c2da..3fe40336 100644 --- a/Modulefile +++ b/Modulefile @@ -1,9 +1,9 @@ name 'puppet-python' -version '1.1.3' +version '1.1.4' -author 'Sergey Stankevich' +author 'Fotis Gimian' license 'Apache License, Version 2.0' -project_page 'https://github.com/stankevich/puppet-python' -source 'git://github.com/stankevich/puppet-python' +project_page 'https://github.com/fgimian/puppet-python' +source 'git://github.com/fgimian/puppet-python' summary 'Puppet module for Python' description 'Installs and manages Python, pip, virtualenv, Gunicorn virtual hosts.' diff --git a/README.md b/README.md index 971bbe87..9e35e36e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ -[puppet-python](https://github.com/stankevich/puppet-python) +[puppet-python](https://github.com/fgimian/puppet-python) ====== -Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. +This project was forked from [stankevich's Github page](https://github.com/stankevich/puppet-python) -**Version 1.1.x Notes** +Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. This fork makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. + +**Version 1.1.x Notes (fgimian fork)** Version 1.1.x makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. @@ -18,7 +20,7 @@ Currently, the changes you need to make are as follows: ``` bash cd /etc/puppet/modules -git clone git://github.com/stankevich/puppet-python.git python +git clone git://github.com/fgimian/puppet-python.git python ``` ## Usage From 006692eceb86079ab4c0976c978d705b160217f0 Mon Sep 17 00:00:00 2001 From: Fotis Gimian Date: Thu, 13 Jun 2013 08:30:19 +1000 Subject: [PATCH 39/59] Reverted documentation to reflect Sergey as the owner now that the pull request was accepted --- Modulefile | 6 +++--- README.md | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Modulefile b/Modulefile index 3fe40336..6bfbe7a4 100644 --- a/Modulefile +++ b/Modulefile @@ -1,9 +1,9 @@ name 'puppet-python' version '1.1.4' -author 'Fotis Gimian' +author 'Sergey Stankevich' license 'Apache License, Version 2.0' -project_page 'https://github.com/fgimian/puppet-python' -source 'git://github.com/fgimian/puppet-python' +project_page 'https://github.com/stankevich/puppet-python' +source 'git://github.com/stankevich/puppet-python' summary 'Puppet module for Python' description 'Installs and manages Python, pip, virtualenv, Gunicorn virtual hosts.' diff --git a/README.md b/README.md index 9e35e36e..971bbe87 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,9 @@ -[puppet-python](https://github.com/fgimian/puppet-python) +[puppet-python](https://github.com/stankevich/puppet-python) ====== -This project was forked from [stankevich's Github page](https://github.com/stankevich/puppet-python) +Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. -Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. This fork makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. - -**Version 1.1.x Notes (fgimian fork)** +**Version 1.1.x Notes** Version 1.1.x makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. @@ -20,7 +18,7 @@ Currently, the changes you need to make are as follows: ``` bash cd /etc/puppet/modules -git clone git://github.com/fgimian/puppet-python.git python +git clone git://github.com/stankevich/puppet-python.git python ``` ## Usage From 291ebd17cdc6ffe16e731bf4ba49cf7e185fc1cf Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Tue, 9 Jul 2013 11:57:38 +0100 Subject: [PATCH 40/59] Install pip before distribute --- manifests/virtualenv.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 354e4d82..039d0853 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -99,7 +99,7 @@ } exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", user => $owner, creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], From e9520954b699fc87c3e58c62bffe027fe11e2f1c Mon Sep 17 00:00:00 2001 From: Frank Chiang Date: Thu, 15 Aug 2013 08:23:48 -0400 Subject: [PATCH 41/59] Change hardcoded path in virtualenv module to take in an optional parameter. --- manifests/virtualenv.pp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 039d0853..14207a5f 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -62,7 +62,8 @@ $owner = 'root', $group = 'root', $proxy = false, - $environment = [] + $environment = [], + $path = [ '/bin', '/usr/bin', '/usr/sbin' ] ) { $venv_dir = $name @@ -102,7 +103,7 @@ command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", user => $owner, creates => "${venv_dir}/bin/activate", - path => [ '/bin', '/usr/bin', '/usr/sbin' ], + path => $path, cwd => "/tmp", environment => $environment, } From 98a51c5885a2f8e27ee41d56e6a8c64de4c9848c Mon Sep 17 00:00:00 2001 From: Frank Chiang Date: Thu, 15 Aug 2013 08:46:29 -0400 Subject: [PATCH 42/59] Add comments describing what path parameter does. --- manifests/virtualenv.pp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 14207a5f..26f50ed4 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -34,6 +34,9 @@ # [*environment*] # Additional environment variables required to install the packages. Default: none # +# [*path*] +# Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ] +# # === Examples # # python::virtualenv { '/var/www/project1': From 12d5427d2856f773427ec17b70cea687afd9c8b1 Mon Sep 17 00:00:00 2001 From: Veros Kaplan Date: Thu, 5 Sep 2013 16:13:36 +0200 Subject: [PATCH 43/59] Added missing Python version --- manifests/virtualenv.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 039d0853..cefffccd 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -99,7 +99,7 @@ } exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} -p ${python} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", user => $owner, creates => "${venv_dir}/bin/activate", path => [ '/bin', '/usr/bin', '/usr/sbin' ], From 4f2c04b70452249e6917859fdbf02bd90ec3af79 Mon Sep 17 00:00:00 2001 From: Antonis Christofides Date: Tue, 8 Oct 2013 17:59:17 +0300 Subject: [PATCH 44/59] Added src parameter for pip in python::requirements A requirements file can have --editable resources; in such a case the pip default is to install them in ~/src/, which can be unsuitable; for example, if run by root, /root/src/ is usually unreadable by other users. pip provides an --src parameter to specify an alternative directory. This is useful for installing requirements systemwide, outside a virtualenv. --- README.md | 2 ++ manifests/requirements.pp | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 971bbe87..7fa9dac4 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ Installs and manages Python packages from requirements file. **owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root +**src** - The ``--src`` parameter to ``pip``, used to specify where to install ``--editable`` resources; by default no ``--src`` parameter is passed to ``pip``. + **group** - The group that was used to create the virtualenv. This is used to create the requirements file with correct permissions if it's not present already. python::requirements { '/var/www/project1/requirements.txt': diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 67906fc8..60c5b6c8 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -19,6 +19,11 @@ # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*src*] +# Pip --src parameter; if the requirements file contains --editable resources, +# this parameter specifies where they will be installed. See the pip +# documentation for more. Default: none (i.e. use the pip default). +# # [*environment*] # Additional environment variables required to install the packages. Default: none # @@ -41,6 +46,7 @@ $owner = 'root', $group = 'root', $proxy = false, + $src = false, $environment = [] ) { @@ -63,6 +69,11 @@ default => "--proxy=${proxy}", } + $src_flag = $src ? { + false => '', + default => "--src=${src}", + } + # This will ensure multiple python::virtualenv definitions can share the # the same requirements file. if !defined(File[$requirements]) { @@ -79,7 +90,7 @@ exec { "python_requirements${name}": provider => shell, - command => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} -r ${requirements}", + command => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} ${src_flag} -r ${requirements}", refreshonly => true, timeout => 1800, user => $owner, From a2e66713c380c81a484583a3a33ff6dead87a496 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Wed, 9 Oct 2013 15:56:13 +0200 Subject: [PATCH 45/59] Add options install_args and uninstall_args. Modern pip does not install what it considers 'pre-release' versions unless '--pre' is given as argument. This change adds a mechanism for supplying the '--pre' argument or any other argument deemed necessary. --- manifests/pip.pp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index ddbcd6fd..1d576e0f 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -35,12 +35,14 @@ # Fotis Gimian # define python::pip ( - $ensure = present, - $virtualenv = 'system', - $url = false, - $owner = 'root', - $proxy = false, - $environment = [] + $ensure = present, + $virtualenv = 'system', + $url = false, + $owner = 'root', + $proxy = false, + $environment = [], + $install_args = '', + $uninstall_args = '', ) { # Parameter validation @@ -80,7 +82,7 @@ case $ensure { present: { exec { "pip_install_${name}": - command => "$pip_env --log-file ${cwd}/pip.log install ${proxy_flag} ${source}", + command => "$pip_env --log-file ${cwd}/pip.log install $install_args ${proxy_flag} ${source}", unless => "$pip_env freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, @@ -89,7 +91,7 @@ default: { exec { "pip_uninstall_${name}": - command => "echo y | $pip_env uninstall ${proxy_flag} ${name}", + command => "echo y | $pip_env uninstall $uninstall_args ${proxy_flag} ${name}", onlyif => "$pip_env freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, From b84a20709beecc417893fdee4e25060ea63fab21 Mon Sep 17 00:00:00 2001 From: Vik Bhatti Date: Tue, 15 Oct 2013 13:38:38 +0100 Subject: [PATCH 46/59] Added support for setting egg name --- manifests/pip.pp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index 1d576e0f..8bd21d38 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -40,6 +40,7 @@ $url = false, $owner = 'root', $proxy = false, + $egg = false, $environment = [], $install_args = '', $uninstall_args = '', @@ -74,9 +75,14 @@ default => "^${name}==", } + $egg_name = $egg ? { + false => $name, + default => $egg + } + $source = $url ? { false => $name, - default => "${url}#egg=${name}", + default => "${url}#egg=${egg_name}", } case $ensure { From f7718f0d9b6184328a301dbcbbd6e5346c8c1f0a Mon Sep 17 00:00:00 2001 From: Vik Bhatti Date: Tue, 15 Oct 2013 15:23:42 +0100 Subject: [PATCH 47/59] Updated docs to include egg param --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7fa9dac4..b8544f42 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ Installs and manages packages from pip. **environment** - Additional environment variables required to install the packages. Default: none +**egg** - The egg name to use. Default: $name of the class, e.g. cx_Oracle + python::pip { 'cx_Oracle': virtualenv => '/var/www/project1', owner => 'appuser', From e7b0e1475daaf778861220835a01c7c6f699a2b8 Mon Sep 17 00:00:00 2001 From: Vik Bhatti Date: Tue, 15 Oct 2013 15:29:37 +0100 Subject: [PATCH 48/59] Updated readme to include install and uninstall args --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b8544f42..a259470a 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ Installs and manages packages from pip. **egg** - The egg name to use. Default: $name of the class, e.g. cx_Oracle +**install_args** - Array of additional flags to pass to pip during installaton. Default: none + +**uninstall_args** - Array of additional flags to pass to pip during uninstall. Default: none + python::pip { 'cx_Oracle': virtualenv => '/var/www/project1', owner => 'appuser', From 31d96b912c16f82aa61d581a443a9b4c52914aa9 Mon Sep 17 00:00:00 2001 From: Vik Bhatti Date: Tue, 15 Oct 2013 15:30:43 +0100 Subject: [PATCH 49/59] Updated python::pip example to include install_args --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a259470a..e9a2b852 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,11 @@ Installs and manages packages from pip. **uninstall_args** - Array of additional flags to pass to pip during uninstall. Default: none python::pip { 'cx_Oracle': - virtualenv => '/var/www/project1', - owner => 'appuser', - proxy => 'http://proxy.domain.com:3128', - environment => 'ORACLE_HOME=/usr/lib/oracle/11.2/client64', + virtualenv => '/var/www/project1', + owner => 'appuser', + proxy => 'http://proxy.domain.com:3128', + environment => 'ORACLE_HOME=/usr/lib/oracle/11.2/client64', + install_args => ['-e'], } ### python::requirements From df25cc78db263ddfbd8588b6cc320a1970387c47 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sat, 19 Oct 2013 15:01:45 +0300 Subject: [PATCH 50/59] Adding .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b2bcbe85 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +pkg/ \ No newline at end of file From 680107d214b452feea02b4ad8dc4fdecaa565d94 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sat, 19 Oct 2013 15:02:25 +0300 Subject: [PATCH 51/59] Version bump --- Modulefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modulefile b/Modulefile index 6bfbe7a4..f8b02b54 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.1.4' +version '1.1.6' author 'Sergey Stankevich' license 'Apache License, Version 2.0' From 94f0993b4f37a7f64bcf54f885f69f9de3a7ccc3 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sat, 19 Oct 2013 16:10:44 +0300 Subject: [PATCH 52/59] Forge upload --- Modulefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modulefile b/Modulefile index f8b02b54..0b1feed6 100644 --- a/Modulefile +++ b/Modulefile @@ -1,4 +1,4 @@ -name 'puppet-python' +name 'stankevich-python' version '1.1.6' author 'Sergey Stankevich' From 65bf6fe8b6cf23050e755d62291f56f7e8654ede Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sat, 19 Oct 2013 16:10:59 +0300 Subject: [PATCH 53/59] pip log option. Fixes #28 --- manifests/pip.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/pip.pp b/manifests/pip.pp index 8bd21d38..3ca33a85 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -88,7 +88,7 @@ case $ensure { present: { exec { "pip_install_${name}": - command => "$pip_env --log-file ${cwd}/pip.log install $install_args ${proxy_flag} ${source}", + command => "$pip_env --log ${cwd}/pip.log install $install_args ${proxy_flag} ${source}", unless => "$pip_env freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, From e47c5bbc602cdc7d0e2c1fffe890517dd190999c Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Sat, 19 Oct 2013 16:14:44 +0300 Subject: [PATCH 54/59] Version 1.1.7 --- Modulefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modulefile b/Modulefile index 0b1feed6..890910f3 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'stankevich-python' -version '1.1.6' +version '1.1.7' author 'Sergey Stankevich' license 'Apache License, Version 2.0' From 8a68085f7450f95ca736c690e14b58a0998a13f1 Mon Sep 17 00:00:00 2001 From: Justin Quick Date: Thu, 7 Nov 2013 15:57:10 -0500 Subject: [PATCH 55/59] added cwd parameter, made timeout configurable --- manifests/virtualenv.pp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index bdcfc6f5..8fc17a97 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -66,7 +66,9 @@ $group = 'root', $proxy = false, $environment = [], - $path = [ '/bin', '/usr/bin', '/usr/sbin' ] + $path = [ '/bin', '/usr/bin', '/usr/sbin' ], + $cwd = undef, + $timeout = 1800 ) { $venv_dir = $name @@ -115,10 +117,11 @@ exec { "python_requirements_initial_install_${requirements}_${venv_dir}": command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} -r ${requirements}", refreshonly => true, - timeout => 1800, + timeout => $timeout, user => $owner, subscribe => Exec["python_virtualenv_${venv_dir}"], environment => $environment, + cwd => $cwd } python::requirements { "${requirements}_${venv_dir}": From c45a37b5819d3c0dab423b93cefb31e65bddad9d Mon Sep 17 00:00:00 2001 From: Justin Quick Date: Thu, 7 Nov 2013 16:03:28 -0500 Subject: [PATCH 56/59] docs --- README.md | 4 ++++ manifests/virtualenv.pp | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index e9a2b852..cea0c2d2 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,10 @@ Creates Python virtualenv. **index** - Base URL of Python package index. Default: none +**cwd** - The directory from which to run the "pip install" command. Default: undef + +**timeout** - The maximum time in seconds the "pip install" command should take. Default: 1800 + python::virtualenv { '/var/www/project1': ensure => present, version => 'system', diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 8fc17a97..84861834 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -37,6 +37,12 @@ # [*path*] # Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ] # +# [*cwd*] +# The directory from which to run the "pip install" command. Default: undef +# +# [*timeout*] +# The maximum time in seconds the "pip install" command should take. Default: 1800 +# # === Examples # # python::virtualenv { '/var/www/project1': From 343788fcd8b64f494145c07c41a39c1afd6eafd6 Mon Sep 17 00:00:00 2001 From: Justin Quick Date: Thu, 7 Nov 2013 16:04:30 -0500 Subject: [PATCH 57/59] example docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cea0c2d2..ec30953c 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ Creates Python virtualenv. distribute => false, owner => 'appuser', group => 'apps', + cwd => '/var/www/project1', + timeout => 0, } ### python::gunicorn From 2d0f8216ab222a533d05d31331267b180c3aac47 Mon Sep 17 00:00:00 2001 From: Justin Quick Date: Thu, 7 Nov 2013 16:05:12 -0500 Subject: [PATCH 58/59] ws fix --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec30953c..0bcd9521 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,8 @@ Creates Python virtualenv. distribute => false, owner => 'appuser', group => 'apps', - cwd => '/var/www/project1', - timeout => 0, + cwd => '/var/www/project1', + timeout => 0, } ### python::gunicorn From 29bf73bfa6d3e0f6b880bb4c4125edc1b8f0675e Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Fri, 8 Nov 2013 13:28:12 +0200 Subject: [PATCH 59/59] Release 1.1.8 --- Modulefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modulefile b/Modulefile index 890910f3..37a5401f 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'stankevich-python' -version '1.1.7' +version '1.1.8' author 'Sergey Stankevich' license 'Apache License, Version 2.0'