diff --git a/.gitignore b/.gitignore index a102ff7..e30868a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .idea *.pyc -*.sqlite3 \ No newline at end of file +*.sqlite3 +static/ +!app/*/static +.vagrant \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..71e9035 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,13 @@ +Vagrant::Config.run do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + config.vm.host_name = "vapemap.local" + config.vm.forward_port 80, 8080 + + config.vm.provision :shell, :inline => "apt-get update --fix-missing" + config.vm.provision :puppet do |puppet| + puppet.manifests_path = "puppet/manifests" + puppet.manifest_file = "vagrant.pp" + puppet.options = ['--templatedir', '/vagrant/puppet/templates'] + end +end diff --git a/app/gunicorn_config b/app/gunicorn_config new file mode 100644 index 0000000..fb34d96 --- /dev/null +++ b/app/gunicorn_config @@ -0,0 +1,8 @@ +#!python +from os import environ +from gevent import monkey +monkey.patch_all() + +bind = "127.0.0.1:3322" +workers = 8 +#worker_class = "gunicorn.workers.ggevent.GeventWorker" diff --git a/app/vapemap/settings.py b/app/vapemap/settings.py index c236850..d1bab9b 100644 --- a/app/vapemap/settings.py +++ b/app/vapemap/settings.py @@ -65,6 +65,7 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django.contrib.admin', 'django.contrib.flatpages', + 'gunicorn', 'south', 'markdown_deux', 'epiceditor', diff --git a/puppet/files/nginx/nginx.conf b/puppet/files/nginx/nginx.conf new file mode 100644 index 0000000..8a8e71d --- /dev/null +++ b/puppet/files/nginx/nginx.conf @@ -0,0 +1,30 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*.conf; +} diff --git a/puppet/files/nginx/nginx.list b/puppet/files/nginx/nginx.list new file mode 100644 index 0000000..4965a03 --- /dev/null +++ b/puppet/files/nginx/nginx.list @@ -0,0 +1 @@ +deb http://nginx.org/packages/ubuntu/ lucid nginx \ No newline at end of file diff --git a/puppet/files/redis/redis.list b/puppet/files/redis/redis.list new file mode 100644 index 0000000..2b5a38f --- /dev/null +++ b/puppet/files/redis/redis.list @@ -0,0 +1 @@ +deb http://ppa.launchpad.net/rwky/redis/ubuntu lucid main \ No newline at end of file diff --git a/puppet/files/uwsgi/uwsgi.init b/puppet/files/uwsgi/uwsgi.init new file mode 100644 index 0000000..8e8ea21 --- /dev/null +++ b/puppet/files/uwsgi/uwsgi.init @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +### BEGIN INIT INFO +# Provides: uwsgi +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: starts the uwsgi app server +# Description: starts uwsgi app server using start-stop-daemon +### END INIT INFO +#set -e + +PATH=/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/usr/local/bin/uwsgi +RUN=/var/run/uwsgi +ENABLED_CONFIGS_DIR=/etc/uwsgi/sites-enabled +AVAILABLE_CONFIGS_DIR=/etc/uwsgi/sites-available +OWNER=www-data +NAME=uwsgi +DESC=uwsgi +OP=$1 + +[[ -x $DAEMON ]] || exit 0 +[[ -d $RUN ]] || mkdir $RUN && chown www-data $RUN + +DAEMON_OPTS="" + +# Include uwsgi defaults if available +if [[ -f /etc/default/uwsgi ]]; then + . /etc/default/uwsgi +fi + +do_pid_check() +{ + local PIDFILE=$1 + [[ -f $PIDFILE ]] || return 0 + local PID=$(cat $PIDFILE) + for p in $(pgrep uwsgi); do + [[ $p == $PID ]] && return 1 + done + return 0 +} + + +do_start() +{ + local PIDFILE=$RUN/uwsgi.pid + local START_OPTS=" \ + --emperor $ENABLED_CONFIGS_DIR \ + --pidfile $PIDFILE \ + --daemonize /var/log/uwsgi/uwsgi-emperor.log \ + " + if do_pid_check $PIDFILE; then + sudo -u $OWNER -i uwsgi $DAEMON_OPTS $START_OPTS + else + echo "Already running!" + fi +} + +send_sig() +{ + local PIDFILE=$RUN/uwsgi.pid + set +e + [[ -f $PIDFILE ]] && kill $1 $(cat $PIDFILE) > /dev/null 2>&1 + set -e +} + +wait_and_clean_pidfile() +{ + local PIDFILE=$RUN/uwsgi.pid + until do_pid_check $PIDFILE; do + echo -n ""; + done + rm -f $PIDFILE +} + +do_stop() +{ + send_sig -3 + wait_and_clean_pidfile +} + +do_reload() +{ + send_sig -1 +} + +do_force_reload() +{ + send_sig -15 +} + +get_status() +{ + send_sig -10 +} + +enable_configs() +{ + local configs + + if [[ $# -eq 0 || ${1,,} = 'all' ]]; then + configs=$(diff $AVAILABLE_CONFIGS_DIR $ENABLED_CONFIGS_DIR \ + | grep $AVAILABLE_CONFIGS_DIR \ + | sed -re 's#.+: (.+)$#\1#') + else + configs=$@ + fi + + for c in $configs; do + echo -n "Enabling $c..." + [[ -f $ENABLED_CONFIGS_DIR/$c ]] && echo "Skipped" && continue + [[ -f $AVAILABLE_CONFIGS_DIR/$c ]] && \ + ln -s $AVAILABLE_CONFIGS_DIR/$c $ENABLED_CONFIGS_DIR && \ + echo "Done" && \ + continue + echo "Error" + done +} + +disable_configs() +{ + local configs + if [[ $# -eq 0 || ${1,,} = 'all' ]]; then + configs=$(find $ENABLED_CONFIGS_DIR -type l -exec basename {} \;) + else + configs=$@ + fi + + for c in $configs; do + local config_path="$ENABLED_CONFIGS_DIR/$c" + echo -n "Disabling $c..." + [[ ! -L $config_path ]] && echo "Skipped" && continue + [[ -f $config_path ]] && rm $config_path && echo "Done" && continue + echo "Error" + done +} + +case "$OP" in + start) + echo "Starting $DESC: " + do_start + echo "$NAME." + ;; + stop) + echo -n "Stopping $DESC: " + do_stop + echo "$NAME." + ;; + reload) + echo -n "Reloading $DESC: " + do_reload + echo "$NAME." + ;; + force-reload) + echo -n "Force-reloading $DESC: " + do_force_reload + echo "$NAME." + ;; + restart) + echo "Restarting $DESC: " + do_stop + sleep 1 + do_start + echo "$NAME." + ;; + status) + get_status + ;; + enable) + shift + enable_configs $@ + ;; + disable) + shift + disable_configs $@ + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|restart|reload|force-reload|status" + "|enable|disable}">&2 + exit 1 + ;; +esac +exit 0 \ No newline at end of file diff --git a/puppet/manifests/classes/apt.pp b/puppet/manifests/classes/apt.pp new file mode 100644 index 0000000..d60e3a3 --- /dev/null +++ b/puppet/manifests/classes/apt.pp @@ -0,0 +1,34 @@ +stage {'repo': before => Stage['pre']} + +class apt { + exec {'apt-get-update': + path => '/usr/local/bin:/usr/bin:/bin', + command => 'apt-get update', + } +} + +class {'apt': stage => 'repo' } + +define apt::repo ($source, $key) { + + file {"/etc/apt/sources.list.d/${name}.list": + source => "${source}", + owner => root, + group => root, + mode => 0644, + notify => Exec['apt-get-update'], + } + + apt::key{"${name}-key": + key => "${key}", + } +} + +define apt::key($key) { + exec {"${name}-exec": + command => "/usr/bin/env bash -c 'apt-key adv --recv-key --keyserver keyserver.ubuntu.com ${key}'", + } +} + +Apt::Repo <| |> -> Exec['apt-get-update'] +Apt::Key <| |> -> Exec['apt-get-update'] \ No newline at end of file diff --git a/puppet/manifests/classes/django.pp b/puppet/manifests/classes/django.pp new file mode 100644 index 0000000..86fcfdd --- /dev/null +++ b/puppet/manifests/classes/django.pp @@ -0,0 +1,6 @@ + +class django::uwsgi { + include nginx + include uwsgi +} + diff --git a/puppet/manifests/classes/gunicorn.pp b/puppet/manifests/classes/gunicorn.pp new file mode 100644 index 0000000..e69de29 diff --git a/puppet/manifests/classes/mysql.pp b/puppet/manifests/classes/mysql.pp new file mode 100644 index 0000000..6ce0ce8 --- /dev/null +++ b/puppet/manifests/classes/mysql.pp @@ -0,0 +1,34 @@ +# Get mysql up and running +class mysql { + package { "mysql-server": + ensure => installed, + } + + case $operatingsystem { + ubuntu: { + package { "libmysqld-dev": + ensure => installed, + } + } + } + + service { "mysql": + ensure => running, + enable => true, + require => Package['mysql-server'], + } +} + +define mysql::database($user, $password) { + exec { "create-${name}-db": + unless => "/usr/bin/mysql -uroot ${name}", + command => "/usr/bin/mysql -uroot -e \"create database ${name};\"", + require => Service["mysql"], + } + + exec { "grant-${name}-db": + unless => "/usr/bin/mysql -u${user} -p${password} ${name}", + command => "/usr/bin/mysql -uroot -e \"grant all on ${name}.* to ${user}@localhost identified by '$password';\"", + require => [Service["mysql"], Exec["create-${name}-db"]] + } +} diff --git a/puppet/manifests/classes/nginx.pp b/puppet/manifests/classes/nginx.pp new file mode 100644 index 0000000..e19efcf --- /dev/null +++ b/puppet/manifests/classes/nginx.pp @@ -0,0 +1,75 @@ +class nginx { + + include nginx::apt + + package {'nginx': + ensure => latest, + require => Apt::Repo['nginx'], + } + service {'nginx': + ensure => running, + enable => true, + require => Package['nginx'], + } + + file{['/etc/nginx/sites-available', '/etc/nginx/sites-enabled']: + ensure => directory, + } + + # Remove the default config + file {['/etc/nginx/sites-enabled/default', '/etc/nginx/conf.d/default.conf']: + ensure => absent, + require => Package['nginx'], + notify => Service['nginx'], + } + + file {'/etc/nginx/nginx.conf': + owner => root, + group => root, + mode => 0644, + ensure => present, + source => '/vagrant/puppet/files/nginx/nginx.conf', + require => Package['nginx'], + notify => Service['nginx'], + } +} + +class nginx::apt { + apt::repo {'nginx': + source => '/vagrant/puppet/files/nginx/nginx.list', + key => 'ABF5BD827BD9BF62', + } +} + +class {'nginx::apt': stage => 'repo' } + + +# Setup a gunicorn instance in nginx +define nginx::gunicorn($ensure, $host, $port, $root, $static='') { + case $ensure { + enabled: { + file{"/etc/nginx/sites-available/${name}.conf": + content => template('nginx/gunicorn.erb'), + ensure => present, + notify => Service['nginx'], + require => [Package['nginx'], File['/etc/nginx/sites-available']], + } + file{"/etc/nginx/sites-enabled/${name}.conf": + ensure => link, + target => "/etc/nginx/sites-available/${name}.conf", + notify => Service['nginx'], + require => [File["/etc/nginx/sites-available/${name}.conf"], File['/etc/nginx/sites-enabled']], + } + } + absent: { + file{"/etc/nginx/sites-available/${name}.conf": + ensure => absent, + notify => Service['nginx'], + } + file{"/etc/nginx/sites-enabled/${name}.conf": + ensure => absent, + notify => Service['nginx'], + } + } + } +} \ No newline at end of file diff --git a/puppet/manifests/classes/python.pp b/puppet/manifests/classes/python.pp new file mode 100644 index 0000000..7b2c903 --- /dev/null +++ b/puppet/manifests/classes/python.pp @@ -0,0 +1,47 @@ +stage { 'pre': before => Stage['main'] } + +class python { + + package { + "build-essential": ensure => latest; + "python": ensure => latest; + "python-dev": ensure => latest; + "python-setuptools": ensure => installed; + "git-core": ensure => installed; + "mercurial": ensure => installed; + "libevent-dev": ensure => installed; + "libgeos-dev": ensure => installed; + } + exec {'pip-package': + command => 'easy_install pip', + path => '/usr/local/bin:/usr/bin:/bin', + require => Package['python-setuptools'], + subscribe => Package['python-setuptools'], + } + package {['virtualenv', 'virtualenvwrapper', 'gunicorn', 'gevent']: + ensure => latest, + provider => pip, + require => Exec['pip-package'], + } +} +class { 'python': stage => 'pre' } + +define python::venv($path, $requirements) { + + file {"$requirements": + ensure => present, + } + + exec{"${name}-venv": + path => '/usr/local/bin:/usr/bin:/bin', + command => "virtualenv --system-site-packages ${path}", + creates => "${path}", + require => Package['virtualenv'], + } + exec {"${name}-requirements": + command => "/usr/bin/env bash -c 'source ${path}/bin/activate; pip install -i http://f.pypi.python.org/simple -r ${requirements}'", + require => Exec["${name}-venv"], + subscribe => File["$requirements"], + } + +} \ No newline at end of file diff --git a/puppet/manifests/classes/redis.pp b/puppet/manifests/classes/redis.pp new file mode 100644 index 0000000..323f341 --- /dev/null +++ b/puppet/manifests/classes/redis.pp @@ -0,0 +1,25 @@ +class redis { + + include redis::apt + + package {'redis-server': + ensure => latest, + require => Apt::Repo['redis'], + } + + service { 'redis-server': + ensure => running, + enable => true, + hasrestart => true, + require => Package['redis-server'], + } +} + +class redis::apt { + apt::repo {'redis': + source => '/vagrant/puppet/files/redis/redis.list', + key => '5862E31D', + } +} + +class {'redis::apt': stage => 'repo' } \ No newline at end of file diff --git a/puppet/manifests/classes/supervisor.pp b/puppet/manifests/classes/supervisor.pp new file mode 100644 index 0000000..81f56e5 --- /dev/null +++ b/puppet/manifests/classes/supervisor.pp @@ -0,0 +1,30 @@ +class supervisor { + package{'supervisor': + ensure => latest, + } + + service{'supervisor': + ensure => running, + hasrestart => false, + require => Package['supervisor'], + } +} + +define supervisor::program( + $command = $title, + $directory = "/tmp/", + $user = "nobody", + $autostart = true, + $autorestart = true, + $redirect_stderr = true) + { + + file {"/etc/supervisor/conf.d/$title.conf": + ensure => present, + owner => "root", + group => "root", + content => template('supervisor/program.erb'), + notify => Service['supervisor'], + require => Package['supervisor'], + } +} \ No newline at end of file diff --git a/puppet/manifests/vagrant.pp b/puppet/manifests/vagrant.pp new file mode 100644 index 0000000..a57bb1d --- /dev/null +++ b/puppet/manifests/vagrant.pp @@ -0,0 +1,59 @@ + +import "classes/*.pp" + +include apt +include python +include nginx +include mysql +include supervisor + +$database = "mysql://vapemap:randompassword1234@localhost/vapemap" + +mysql::database{'vapemap': + user => 'vapemap', + password => 'randompassword1234' +} + +python::venv {'vapemap': + path => "/usr/local/${name}-venv", + requirements => '/vagrant/requirements.txt', +} + +file{'/usr/local/bin/vapemap-init.sh': + content => "#!/bin/bash\n. /usr/local/main-venv/bin/activate\nDATABASE_URL=$database /usr/bin/env python manage.py run_gunicorn -c gunicorn_config --preload", + ensure => present, + mode => 755, +} + +supervisor::program {'vapemap': + command => '/usr/local/bin/vapemap-init.sh', + directory => '/vagrant/app/', + user => 'vagrant', + require => [File['/usr/local/bin/vapemap-init.sh'], Python::Venv['vapemap'], Mysql::Database['vapemap']], +} + +nginx::gunicorn { 'vapemap': + ensure => enabled, + host => '_', + port => 3322, + root => '/vagrant/root/', + static => '/vagrant/static/', +} + +exec{'vapemap-collectstatic': + command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py collectstatic --noinput'", + path => '/usr/local/bin:/usr/bin:/bin', + require => Python::Venv['vapemap'], +} + +exec{'vapemap-syncdb': + command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py syncdb --all --noinput'", + path => '/usr/local/bin:/usr/bin:/bin', + require => [Python::Venv['vapemap'], Mysql::Database['vapemap']], +} + +exec{'vapemap-migrationfake': + command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py migrate --fake --noinput'", + path => '/usr/local/bin:/usr/bin:/bin', + require => Exec['vapemap-syncdb'], +} \ No newline at end of file diff --git a/puppet/templates/nginx/gunicorn.erb b/puppet/templates/nginx/gunicorn.erb new file mode 100644 index 0000000..d33cee5 --- /dev/null +++ b/puppet/templates/nginx/gunicorn.erb @@ -0,0 +1,23 @@ +server { + listen 80; + server_name <%= @host %>; + server_name_in_redirect off; + + <% if @static %> + location /static { + alias <%= @static %>; + } + <% end %> + + location / { + proxy_pass_header Server; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 10; + proxy_read_timeout 3600; + proxy_pass http://127.0.0.1:<%= @port %>/; + proxy_buffering off; + } +} \ No newline at end of file diff --git a/puppet/templates/python/gunicorn.erb b/puppet/templates/python/gunicorn.erb new file mode 100644 index 0000000..ad8e3dd --- /dev/null +++ b/puppet/templates/python/gunicorn.erb @@ -0,0 +1,35 @@ +CONFIG = { +<% if mode == 'django' -%> + 'mode': 'django', +<% else -%> + 'mode': 'wsgi', +<% end -%> +<% if virtualenv -%> + 'environment': { +<% if environment -%> + 'ENVIRONMENT': '<%= environment %>', +<% end -%> + 'PYTHONPATH': '<%= virtualenv %>' + }, +<% end -%> + 'working_dir': '<%= dir %>', + 'user': 'www-data', + 'group': 'www-data', +<% if virtualenv -%> + 'python': '<%= virtualenv %>/bin/python', +<% else -%> + 'python': '/usr/bin/python', +<% end -%> + 'args': ( +<% if !virtualenv and !bind -%> + '--bind=unix:/tmp/gunicorn-<%= name %>.socket', +<% elsif virtualenv and !bind -%> + '--bind=unix:<%= virtualenv %>/<%= name %>.socket', +<% else -%> + '--bind=<%= bind %>', +<% end -%> + '--workers=<%= @processorcount.to_i*2 %>', + '--timeout=30', + 'app:app', + ), +} \ No newline at end of file diff --git a/puppet/templates/supervisor/program.erb b/puppet/templates/supervisor/program.erb new file mode 100644 index 0000000..5adec17 --- /dev/null +++ b/puppet/templates/supervisor/program.erb @@ -0,0 +1,7 @@ +[program:<%= @title %>] +command=<%= @command %> +directory=<%= @directory %> +user=<%= @user %> +autostart=<%= @autostart %> +autorestart=<%= @autorestart %> +redirect_stderr=<%= @redirect_stderr %> \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 23cc60a..7282ea2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ geopy>=0.95 django-markdown-deux>=1.0.4 django-bootstrap-form>=1.4 django-extra-views>=0.6.2 --e hg+https://bitbucket.org/ubernostrum/django-registration@b3c41b3#egg=django_registration +-e hg+https://bitbucket.org/ubernostrum/django-registration@89093bc#egg=django_registration requests pyelasticsearch -e git+git://github.com/toastdriven/django-haystack.git@0e8bd20c18ce3133b3a4f285a4c420bb621ac49b#egg=django_haystack