Vagrant Support

Brings in the basic config for Vagrant, missing elasticsearch but its enough to bring up a Vagrant instance of the main web app.

.gitignore

@@ -1,3 +1,6 @@
 .idea
 *.pyc
-*.sqlite3
+*.sqlite3
+static/
+!app/*/static
+.vagrant

Vagrantfile

@@ -0,0 +1,13 @@
+Vagrant::Config.run do |config|
+  config.vm.box = "precise64"
+  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.host_name = "vapemap.local"
+  config.vm.forward_port 80, 8080
+
+  config.vm.provision :shell, :inline => "apt-get update --fix-missing"
+  config.vm.provision :puppet do |puppet|
+      puppet.manifests_path = "puppet/manifests"
+      puppet.manifest_file  = "vagrant.pp"
+      puppet.options = ['--templatedir', '/vagrant/puppet/templates']
+  end
+end

app/gunicorn_config

@@ -0,0 +1,8 @@
+#!python
+from os import environ
+from gevent import monkey
+monkey.patch_all()
+
+bind = "127.0.0.1:3322"
+workers = 8 
+#worker_class = "gunicorn.workers.ggevent.GeventWorker"

app/vapemap/settings.py

@@ -65,6 +65,7 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'django.contrib.admin',
     'django.contrib.flatpages',
+    'gunicorn',
     'south',
     'markdown_deux',
     'epiceditor',

puppet/files/nginx/nginx.conf

@@ -0,0 +1,30 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+    include /etc/nginx/sites-enabled/*.conf;
+}

puppet/files/nginx/nginx.list

@@ -0,0 +1 @@
+deb http://nginx.org/packages/ubuntu/ lucid nginx

puppet/files/redis/redis.list

@@ -0,0 +1 @@
+deb http://ppa.launchpad.net/rwky/redis/ubuntu lucid main

puppet/files/uwsgi/uwsgi.init

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+
+### BEGIN INIT INFO
+# Provides:          uwsgi
+# Required-Start:    $all
+# Required-Stop:     $all
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: starts the uwsgi app server
+# Description:       starts uwsgi app server using start-stop-daemon
+### END INIT INFO
+#set -e
+
+PATH=/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON=/usr/local/bin/uwsgi
+RUN=/var/run/uwsgi
+ENABLED_CONFIGS_DIR=/etc/uwsgi/sites-enabled
+AVAILABLE_CONFIGS_DIR=/etc/uwsgi/sites-available
+OWNER=www-data
+NAME=uwsgi
+DESC=uwsgi
+OP=$1
+
+[[ -x $DAEMON ]] || exit 0
+[[ -d $RUN ]] || mkdir $RUN && chown www-data $RUN
+
+DAEMON_OPTS=""
+
+# Include uwsgi defaults if available
+if [[ -f /etc/default/uwsgi ]]; then
+    . /etc/default/uwsgi
+fi
+
+do_pid_check()
+{
+    local PIDFILE=$1
+    [[ -f $PIDFILE ]] || return 0
+    local PID=$(cat $PIDFILE)
+    for p in $(pgrep uwsgi); do
+        [[ $p == $PID ]] && return 1
+    done
+    return 0
+}
+
+
+do_start()
+{
+    local PIDFILE=$RUN/uwsgi.pid
+    local START_OPTS=" \
+        --emperor $ENABLED_CONFIGS_DIR \
+        --pidfile $PIDFILE \
+        --daemonize /var/log/uwsgi/uwsgi-emperor.log \
+        "
+    if do_pid_check $PIDFILE; then
+        sudo -u $OWNER -i uwsgi $DAEMON_OPTS $START_OPTS
+    else
+        echo "Already running!"
+    fi
+}
+
+send_sig()
+{
+    local PIDFILE=$RUN/uwsgi.pid
+    set +e
+    [[ -f $PIDFILE ]] && kill $1 $(cat $PIDFILE) > /dev/null 2>&1
+    set -e
+}
+
+wait_and_clean_pidfile()
+{
+    local PIDFILE=$RUN/uwsgi.pid
+    until do_pid_check $PIDFILE; do
+        echo -n "";
+    done
+    rm -f $PIDFILE
+}
+
+do_stop()
+{
+    send_sig -3
+    wait_and_clean_pidfile
+}
+
+do_reload()
+{
+    send_sig -1
+}
+
+do_force_reload()
+{
+    send_sig -15
+}
+
+get_status()
+{
+    send_sig -10
+}
+
+enable_configs()
+{
+    local configs
+
+    if [[ $# -eq 0 || ${1,,} = 'all' ]]; then
+        configs=$(diff $AVAILABLE_CONFIGS_DIR $ENABLED_CONFIGS_DIR \
+            | grep $AVAILABLE_CONFIGS_DIR \
+            | sed -re 's#.+: (.+)$#\1#')
+    else
+        configs=$@
+    fi
+
+    for c in $configs; do
+        echo -n "Enabling $c..."
+        [[ -f $ENABLED_CONFIGS_DIR/$c ]] && echo "Skipped" && continue
+        [[ -f $AVAILABLE_CONFIGS_DIR/$c ]] && \
+            ln -s $AVAILABLE_CONFIGS_DIR/$c $ENABLED_CONFIGS_DIR && \
+            echo "Done" && \
+            continue
+        echo "Error"
+    done
+}
+
+disable_configs()
+{
+    local configs
+    if [[ $# -eq 0 || ${1,,} = 'all' ]]; then
+        configs=$(find $ENABLED_CONFIGS_DIR -type l -exec basename {} \;)
+    else
+        configs=$@
+    fi
+
+    for c in $configs; do
+        local config_path="$ENABLED_CONFIGS_DIR/$c"
+        echo -n "Disabling $c..."
+        [[ ! -L $config_path ]] && echo "Skipped" && continue
+        [[ -f $config_path ]] && rm $config_path && echo "Done" && continue
+        echo "Error"
+    done
+}
+
+case "$OP" in
+    start)
+        echo "Starting $DESC: "
+        do_start
+        echo "$NAME."
+        ;;
+    stop)
+        echo -n "Stopping $DESC: "
+        do_stop
+        echo "$NAME."
+        ;;
+    reload)
+        echo -n "Reloading $DESC: "
+        do_reload
+        echo "$NAME."
+        ;;
+    force-reload)
+        echo -n "Force-reloading $DESC: "
+        do_force_reload
+        echo "$NAME."
+       ;;
+    restart)
+        echo  "Restarting $DESC: "
+        do_stop
+        sleep 1
+        do_start
+        echo "$NAME."
+        ;;
+    status)
+        get_status
+        ;;
+    enable)
+        shift
+        enable_configs $@
+        ;;
+    disable)
+        shift
+        disable_configs $@
+        ;;
+    *)
+        N=/etc/init.d/$NAME
+        echo "Usage: $N {start|stop|restart|reload|force-reload|status"
+            "|enable|disable}">&2
+        exit 1
+        ;;
+esac
+exit 0

puppet/manifests/classes/apt.pp

@@ -0,0 +1,34 @@
+stage {'repo': before => Stage['pre']}
+
+class apt {
+  exec {'apt-get-update':
+    path => '/usr/local/bin:/usr/bin:/bin',
+    command => 'apt-get update',
+  }
+}
+
+class {'apt': stage => 'repo' }
+
+define apt::repo ($source, $key) {
+
+  file {"/etc/apt/sources.list.d/${name}.list":
+    source => "${source}",
+    owner => root,
+    group => root,
+    mode => 0644,
+    notify => Exec['apt-get-update'],
+  }
+
+  apt::key{"${name}-key":
+    key => "${key}",
+  }
+}
+
+define apt::key($key) {
+  exec {"${name}-exec":
+    command => "/usr/bin/env bash -c 'apt-key adv --recv-key --keyserver keyserver.ubuntu.com ${key}'",
+  }
+}
+
+Apt::Repo <| |> -> Exec['apt-get-update']
+Apt::Key <| |> -> Exec['apt-get-update']

puppet/manifests/classes/django.pp

@@ -0,0 +1,6 @@
+
+class django::uwsgi {
+    include nginx
+    include uwsgi
+}
+

puppet/manifests/classes/mysql.pp

@@ -0,0 +1,34 @@
+# Get mysql up and running
+class mysql {
+  package { "mysql-server":
+    ensure => installed,
+  }
+
+  case $operatingsystem {
+    ubuntu: {
+      package { "libmysqld-dev":
+        ensure => installed,
+      }
+    }
+  }
+
+  service { "mysql":
+    ensure => running,
+    enable => true,
+    require => Package['mysql-server'],
+  }
+}
+
+define mysql::database($user, $password) {
+  exec { "create-${name}-db":
+    unless => "/usr/bin/mysql -uroot ${name}",
+    command => "/usr/bin/mysql -uroot -e \"create database ${name};\"",
+    require => Service["mysql"],
+  }
+
+  exec { "grant-${name}-db":
+    unless => "/usr/bin/mysql -u${user} -p${password} ${name}",
+    command => "/usr/bin/mysql -uroot -e \"grant all on ${name}.* to ${user}@localhost identified by '$password';\"",
+    require => [Service["mysql"], Exec["create-${name}-db"]]
+  }
+}

puppet/manifests/classes/nginx.pp

@@ -0,0 +1,75 @@
+class nginx {
+
+  include nginx::apt
+
+  package {'nginx':
+    ensure => latest,
+    require => Apt::Repo['nginx'],
+  }
+  service {'nginx':
+    ensure => running,
+    enable => true,
+    require => Package['nginx'],
+  }
+
+  file{['/etc/nginx/sites-available', '/etc/nginx/sites-enabled']:
+    ensure => directory,
+  }
+
+  # Remove the default config
+  file {['/etc/nginx/sites-enabled/default', '/etc/nginx/conf.d/default.conf']:
+    ensure => absent,
+    require => Package['nginx'],
+    notify => Service['nginx'],
+  }
+
+  file {'/etc/nginx/nginx.conf':
+    owner => root,
+    group => root,
+    mode => 0644,
+    ensure => present,
+    source => '/vagrant/puppet/files/nginx/nginx.conf',
+    require => Package['nginx'],
+    notify => Service['nginx'],
+  }
+}
+
+class nginx::apt {
+  apt::repo {'nginx':
+    source => '/vagrant/puppet/files/nginx/nginx.list',
+    key => 'ABF5BD827BD9BF62',
+  }
+}
+
+class {'nginx::apt': stage => 'repo' }
+
+
+# Setup a gunicorn instance in nginx
+define nginx::gunicorn($ensure, $host, $port, $root, $static='') {
+  case $ensure {
+    enabled: {
+      file{"/etc/nginx/sites-available/${name}.conf":
+        content => template('nginx/gunicorn.erb'),
+        ensure => present,
+        notify => Service['nginx'],
+        require => [Package['nginx'], File['/etc/nginx/sites-available']],
+      }
+      file{"/etc/nginx/sites-enabled/${name}.conf":
+        ensure => link,
+        target => "/etc/nginx/sites-available/${name}.conf",
+        notify => Service['nginx'],
+        require => [File["/etc/nginx/sites-available/${name}.conf"], File['/etc/nginx/sites-enabled']],
+      }
+    }
+    absent: {
+      file{"/etc/nginx/sites-available/${name}.conf":
+        ensure => absent,
+        notify => Service['nginx'],
+      }
+      file{"/etc/nginx/sites-enabled/${name}.conf":
+        ensure => absent,
+        notify => Service['nginx'],
+      }
+    }
+  }
+}

puppet/manifests/classes/python.pp

@@ -0,0 +1,47 @@
+stage { 'pre': before => Stage['main'] }
+
+class python {
+
+  package {
+    "build-essential": ensure => latest;
+    "python": ensure => latest;
+    "python-dev": ensure => latest;
+    "python-setuptools": ensure => installed;
+    "git-core": ensure => installed;
+    "mercurial": ensure => installed;
+    "libevent-dev": ensure => installed;
+    "libgeos-dev": ensure => installed;
+  }
+  exec {'pip-package':
+    command => 'easy_install pip',
+    path => '/usr/local/bin:/usr/bin:/bin',
+    require => Package['python-setuptools'],
+    subscribe => Package['python-setuptools'],
+  }
+  package {['virtualenv', 'virtualenvwrapper', 'gunicorn', 'gevent']:
+    ensure => latest,
+    provider => pip,
+    require => Exec['pip-package'],
+  }
+}
+class { 'python': stage => 'pre' }
+
+define python::venv($path, $requirements) {
+
+  file {"$requirements":
+    ensure => present,
+  }
+
+  exec{"${name}-venv":
+    path => '/usr/local/bin:/usr/bin:/bin',
+    command => "virtualenv --system-site-packages ${path}",
+    creates => "${path}",
+    require => Package['virtualenv'],
+  }
+  exec {"${name}-requirements":
+    command => "/usr/bin/env bash -c 'source ${path}/bin/activate; pip install -i http://f.pypi.python.org/simple -r ${requirements}'",
+    require => Exec["${name}-venv"],
+    subscribe => File["$requirements"],
+  }
+
+}

puppet/manifests/classes/redis.pp

@@ -0,0 +1,25 @@
+class redis {
+
+  include redis::apt
+
+  package {'redis-server':
+    ensure => latest,
+    require => Apt::Repo['redis'],
+  }
+
+  service { 'redis-server':
+    ensure => running,
+    enable => true,
+    hasrestart => true,
+    require => Package['redis-server'],
+  }
+}
+
+class redis::apt {
+  apt::repo {'redis':
+    source => '/vagrant/puppet/files/redis/redis.list',
+    key => '5862E31D',
+  }
+}
+
+class {'redis::apt': stage => 'repo' }

puppet/manifests/classes/supervisor.pp

@@ -0,0 +1,30 @@
+class supervisor {
+  package{'supervisor':
+    ensure => latest,
+  }
+
+  service{'supervisor':
+    ensure => running,
+    hasrestart => false,
+    require => Package['supervisor'],
+  }
+}
+
+define supervisor::program(
+  $command         = $title,
+  $directory       = "/tmp/",
+  $user            = "nobody",
+  $autostart       = true,
+  $autorestart     = true,
+  $redirect_stderr = true)
+  {
+
+  file {"/etc/supervisor/conf.d/$title.conf":
+    ensure => present,
+    owner => "root",
+    group => "root",
+    content => template('supervisor/program.erb'),
+    notify => Service['supervisor'],
+    require => Package['supervisor'],
+  }
+}

puppet/manifests/vagrant.pp

@@ -0,0 +1,59 @@
+
+import "classes/*.pp"
+
+include apt
+include python
+include nginx
+include mysql
+include supervisor
+
+$database = "mysql://vapemap:randompassword1234@localhost/vapemap"
+
+mysql::database{'vapemap':
+  user => 'vapemap',
+  password => 'randompassword1234'
+}
+
+python::venv {'vapemap':
+  path => "/usr/local/${name}-venv",
+  requirements => '/vagrant/requirements.txt',
+}
+
+file{'/usr/local/bin/vapemap-init.sh':
+  content => "#!/bin/bash\n. /usr/local/main-venv/bin/activate\nDATABASE_URL=$database /usr/bin/env python manage.py run_gunicorn -c gunicorn_config --preload",
+  ensure => present,
+  mode => 755,
+}
+
+supervisor::program {'vapemap':
+  command => '/usr/local/bin/vapemap-init.sh',
+  directory => '/vagrant/app/',
+  user => 'vagrant',
+  require => [File['/usr/local/bin/vapemap-init.sh'], Python::Venv['vapemap'], Mysql::Database['vapemap']],
+}
+
+nginx::gunicorn { 'vapemap':
+  ensure => enabled,
+  host => '_',
+  port => 3322,
+  root => '/vagrant/root/',
+  static => '/vagrant/static/',
+}
+
+exec{'vapemap-collectstatic':
+  command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py collectstatic --noinput'",
+  path => '/usr/local/bin:/usr/bin:/bin',
+  require => Python::Venv['vapemap'],
+}
+
+exec{'vapemap-syncdb':
+  command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py syncdb --all --noinput'",
+  path => '/usr/local/bin:/usr/bin:/bin',
+  require => [Python::Venv['vapemap'], Mysql::Database['vapemap']],
+}
+
+exec{'vapemap-migrationfake':
+  command => "/usr/bin/env bash -c 'source /usr/local/main-venv/bin/activate; cd /vagrant/app;DATABASE_URL=$database /usr/bin/env python ./manage.py migrate --fake --noinput'",
+  path => '/usr/local/bin:/usr/bin:/bin',
+  require => Exec['vapemap-syncdb'],
+}

puppet/templates/nginx/gunicorn.erb

@@ -0,0 +1,23 @@
+server {
+    listen 80;
+    server_name <%= @host %>;
+    server_name_in_redirect off;
+
+    <% if @static %>
+    location /static {
+        alias <%= @static %>;
+    }
+    <% end %>
+
+    location / {
+        proxy_pass_header Server;
+        proxy_set_header Host $http_host;
+        proxy_redirect off;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Scheme $scheme;
+        proxy_connect_timeout 10;
+        proxy_read_timeout 3600;
+        proxy_pass http://127.0.0.1:<%= @port %>/;
+        proxy_buffering off;
+    }
+}

puppet/templates/python/gunicorn.erb

@@ -0,0 +1,35 @@
+CONFIG = {
+<% if mode == 'django' -%>
+  'mode': 'django',
+<% else -%>
+  'mode': 'wsgi',
+<% end -%>
+<% if virtualenv -%>
+  'environment': {
+<% if environment -%>
+    'ENVIRONMENT': '<%= environment %>',
+<% end -%>
+    'PYTHONPATH': '<%= virtualenv %>'
+  },
+<% end -%>
+  'working_dir': '<%= dir %>',
+  'user': 'www-data',
+  'group': 'www-data',
+<% if virtualenv -%>
+  'python': '<%= virtualenv %>/bin/python',
+<% else -%>
+  'python': '/usr/bin/python',
+<% end -%>
+  'args': (
+<% if !virtualenv and !bind -%>
+    '--bind=unix:/tmp/gunicorn-<%= name %>.socket',
+<% elsif virtualenv and !bind -%>
+    '--bind=unix:<%= virtualenv %>/<%= name %>.socket',
+<% else -%>
+    '--bind=<%= bind %>',
+<% end -%>
+    '--workers=<%= @processorcount.to_i*2 %>',
+    '--timeout=30',
+    'app:app',
+  ),
+}

puppet/templates/supervisor/program.erb

@@ -0,0 +1,7 @@
+[program:<%= @title %>]
+command=<%= @command %>
+directory=<%= @directory %>
+user=<%= @user %>
+autostart=<%= @autostart %>
+autorestart=<%= @autorestart %>
+redirect_stderr=<%= @redirect_stderr %>

requirements.txt

@@ -5,7 +5,7 @@ geopy>=0.95
 django-markdown-deux>=1.0.4
 django-bootstrap-form>=1.4
 django-extra-views>=0.6.2
--e hg+https://bitbucket.org/ubernostrum/django-registration@b3c41b3#egg=django_registration
+-e hg+https://bitbucket.org/ubernostrum/django-registration@89093bc#egg=django_registration
 requests
 pyelasticsearch
 -e git+git://github.com/toastdriven/django-haystack.git@0e8bd20c18ce3133b3a4f285a4c420bb621ac49b#egg=django_haystack