initial docker deployment implementation

deploy/docker/README.md

@@ -0,0 +1,3 @@
+# MediaCMS on Docker
+
+See: [Details](../../docs/Docker_deployment.md)

deploy/docker/local_settings.py

@@ -0,0 +1,32 @@
+FRONTEND_HOST = 'http://localhost'
+PORTAL_NAME = 'MediaCMS'
+SECRET_KEY = 'ma!s3^b-cw!f#7s6s0m3*jx77a@riw(7701**(r=ww%w!2+yk2'
+POSTGRES_HOST = 'db'
+REDIS_LOCATION = "redis://redis:6379/1"
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "mediacms",
+        "HOST": POSTGRES_HOST,
+        "PORT": "5432",
+        "USER": "mediacms",
+        "PASSWORD": "mediacms",
+    }
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_LOCATION,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
+
+# CELERY STUFF
+BROKER_URL = REDIS_LOCATION
+CELERY_RESULT_BACKEND = BROKER_URL
+
+DEBUG = False

deploy/docker/nginx_http_only.conf

@@ -0,0 +1,30 @@
+server {
+    listen 80 ;
+
+    gzip on;
+    access_log /var/log/nginx/mediacms.io.access.log;
+
+    error_log  /var/log/nginx/mediacms.io.error.log  warn;
+
+    location /static {
+        alias /home/mediacms.io/mediacms/static ;
+    }
+
+    location /media/original {
+        alias /home/mediacms.io/mediacms/media_files/original;
+    }
+
+    location /media {
+        alias /home/mediacms.io/mediacms/media_files ;
+    }
+
+    location / {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+        add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+
+        include /etc/nginx/sites-enabled/uwsgi_params;
+        uwsgi_pass 127.0.0.1:9000;
+    }
+}

deploy/docker/prestart.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+mkdir -p /home/mediacms.io/mediacms/logs
+touch /home/mediacms.io/mediacms/logs/debug.log
+chown www-data. -R /home/mediacms.io/mediacms/logs
+
+RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz0123456789';print(''.join(secrets.choice(chars) for i in range(10)))"`
+ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS}
+
+if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then
+    python manage.py migrate
+    python manage.py loaddata fixtures/encoding_profiles.json
+    python manage.py loaddata fixtures/categories.json
+    python manage.py collectstatic --noinput
+
+    echo "Admin Password: $ADMIN_PASSWORD"
+
+    # post_save, needs redis to succeed (ie. migrate depends on redis)
+    DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
+        --no-input \
+        --username=$ADMIN_USER \
+        --email=$ADMIN_EMAIL \
+        --database=default || true
+
+    # echo "Updating hostname ..."
+    # TODO: Get the FRONTEND_HOST from cms/local_settings.py
+    # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell
+fi
+
+# Setting up internal nginx server
+# HTTPS setup is delegated to a reverse proxy running infront of the application
+
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-available/default
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-enabled/default
+cp deploy/docker/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
+cp deploy/docker/nginx.conf /etc/nginx/
+
+#### Supervisord Configurations #####
+
+cp deploy/docker/supervisord/supervisord-debian.conf /etc/supervisor/conf.d/supervisord-debian.conf
+
+if [ X"$ENABLE_UWSGI" = X"yes" ] ; then
+    echo "Enabling uwsgi app server"
+    cp deploy/docker/supervisord/supervisord-uwsgi.conf /etc/supervisor/conf.d/supervisord-uwsgi.conf
+fi
+
+if [ X"$ENABLE_NGINX" = X"yes" ] ; then
+    echo "Enabling nginx as uwsgi app proxy and media server"
+    cp deploy/docker/supervisord/supervisord-nginx.conf /etc/supervisor/conf.d/supervisord-nginx.conf
+fi
+
+if [ X"$ENABLE_CELERY_BEAT" = X"yes" ] ; then
+    echo "Enabling celery-beat scheduling server"
+    cp deploy/docker/supervisord/supervisord-celery_beat.conf /etc/supervisor/conf.d/supervisord-celery_beat.conf
+fi
+
+if [ X"$ENABLE_CELERY_SHORT" = X"yes" ] ; then
+    echo "Enabling celery-short task worker"
+    cp deploy/docker/supervisord/supervisord-celery_short.conf /etc/supervisor/conf.d/supervisord-celery_short.conf
+fi
+
+if [ X"$ENABLE_CELERY_LONG" = X"yes" ] ; then
+    echo "Enabling celery-long task worker"
+    cp deploy/docker/supervisord/supervisord-celery_long.conf /etc/supervisor/conf.d/supervisord-celery_long.conf
+fi

deploy/docker/reverse_proxy/certs/mediacms.io.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIJAPHG6VrZeH1/MA0GCSqGSIb3DQEBBQUAMBYxFDASBgNV
+BAMTC21lZGlhY21zLmlvMB4XDTIxMDExNjE1NDUzNVoXDTMxMDExNDE1NDUzNVow
+FjEUMBIGA1UEAxMLbWVkaWFjbXMuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC+xxnUwjoIZq8sTw2DYGgIYxQ5lJ8Uvt+z+K/PQpT5nFqd1jURF8Zd
+a92TlJjM5aSKosavuPHbFNkA7rSnLvP+I+8qsNPoinEUlE12Spg4E7dQkOkvTGty
+/amFq69o9vm46GpvwImTZ5AQkzejk0ARUuFSdq9ev4aA44IBYiV4c2jRqnP7LY4j
++SA/rt+9bNUTwQ6QWEHDTHmKePr91UTZBcDw/oaoaJwWFXuEVC7VjtDN09ZNjkdg
+pI6PvQZVw2IlBHS4S+ol+G2k2ckSCLgOj+dZrndr8OGrlAb8wgsInLK54nHm6VRe
+G883CJd/VlOQAulE26ZkzIdAIjJCwb+DAgMBAAGjGjAYMBYGA1UdEQQPMA2CC21l
+ZGlhY21zLmlvMA0GCSqGSIb3DQEBBQUAA4IBAQBwxkTE5GBuFjcFsBzMqhePgC7W
+INzoTmyMLJrNClFLkUKkDrwNmShLNhZUbMHeDD1W40aKYJCV44QhT04fK18HU/DW
+RkprlJDI8WUnuY97zN6Ms9z/GwYDGNXGLh8I/SEMhfJ8cIQuofhvuyi/E4AdWRva
+Hw1RSC8RikTZQ5Y84oJ44RfHNfK7xkaeurcm/Tn4Vxx4RgXA2MMoFA7XbT08vhKw
+iiQ9u4QL1GP3Nm8cTDDA9OChhLl56k24MD3WJM2HFTFlE5S4hFRkEqzy4pI/BTU4
+S4fkXK88xDtB/kHlHgRQiNH+6ik8ZXXP1F56+vDLuR28nK3hRTpQwaRQ7dzC
+-----END CERTIFICATE-----

deploy/docker/reverse_proxy/certs/mediacms.io.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvscZ1MI6CGavLE8Ng2BoCGMUOZSfFL7fs/ivz0KU+ZxandY1
+ERfGXWvdk5SYzOWkiqLGr7jx2xTZAO60py7z/iPvKrDT6IpxFJRNdkqYOBO3UJDp
+L0xrcv2phauvaPb5uOhqb8CJk2eQEJM3o5NAEVLhUnavXr+GgOOCAWIleHNo0apz
++y2OI/kgP67fvWzVE8EOkFhBw0x5inj6/dVE2QXA8P6GqGicFhV7hFQu1Y7QzdPW
+TY5HYKSOj70GVcNiJQR0uEvqJfhtpNnJEgi4Do/nWa53a/Dhq5QG/MILCJyyueJx
+5ulUXhvPNwiXf1ZTkALpRNumZMyHQCIyQsG/gwIDAQABAoIBAQCMauVTWOX3+wRi
+G4l5skLAMZTYUNDKJzdmMtvMNFiMZI258Mk8XIBvkI4VKuFQppH2TJrrCbhSJUUX
+z5p+FywVWYOWq3I9jXBv0jw1ne/uDmz1ysMnQhswFw5oSZahLm9drwtwV4mrSiWa
+XZEtP5t/ZL5dwOeRWGz6fvnYZNHpldkyZDO8+ywB55P+XDfGyxUppMOWdbNV9wGo
+Fg6ypUFWFEUD8Ou8xd5FT5QqrQ5ruZJDKcYhPTuK0/dRsMgAxB+Bhf8XH3ynUZp2
++qMXcKyIQumq9r+/ulE/Yhnbh/E4hYBbThhnmPejNeSvWb7niYfL/fsPI8FLmtmi
+z+Ab5IABAoGBAPb4rUP3rVDatzsf2jJSUXcMn9gAdf3ajbw+Z2CCf4j1wj1BTIM5
+5YmABJMS7D97H+a6Vn+SZd426UJYMlKPDnVOTXvvlzhP/TmiSFa8FW8Rjho9Rcnc
+LDwnO48q0AJg3HslrjEUaDuWaNHJkqB5tGqzKgZCZxrqoNRYgufPwH+DAoGBAMXA
+hr4KxwlcXYIwbM+Uj8eHnESwPWk2+cRwpv62u4ezctZrBCHgAHKvznG7VUeniQfj
+P2MaGFz6Pvzw3cFRLKRVqJom5iXO6+H0EucusdqJY4xdWZt02ZweoJXZi9tiDGmG
+fPOp3vUax4uGUS4LeSo+ZhPNfbfy9c8ZGQ7Z9cABAoGAA0oyvKoK8/3F3RLCjFMO
+ZMCVTIJNEBGeO7i1FdMHMeLcMIazJzhZN2iuJutknD/en+sxhceEdd5TYx/bo7/m
+GGfvnkwFvqlKHT9tKUKeInmgY/cW++Zj7HU1VOXkGXQC290Xoe28qbaKNOkze9HD
+NnymfajayMABXnLDY6Uf0lMCgYBgVLIOn4dnuvPeOKK42ADWTOxF1aiEuYAgPlRL
+Hk7qAvN9GfKQYeM1+whRBNW9KxKoof290/dsS4clhlwwEM/zWbrhJPPWFR95GYGf
+1nJTJ7wzo0HEZb6fu5e0h54Gh5POT/JMbEKtGZd9Ezg2euZSOsVU/jQwyI0PjoVT
+Y7/AAQKBgHEWxqZwJ3BE9gXRRGhKiPLNG9+OzwjrMiinN4s9Hol0STB1AqryswpU
+9QL7Mb8SdxCDY2CseQRwT2VFiP10ElCuzZJ6Yk2cxzmrxAAhRWGHSIuNlnXTA6LR
+AhIMLFLz+7KqBx8VHybkhZNCR1nPR9MicS9MpSEYTuRAnV2B1cuU
+-----END RSA PRIVATE KEY-----

deploy/docker/reverse_proxy/client_max_body_size.conf

@@ -0,0 +1 @@
+client_max_body_size 1g;

deploy/docker/start.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env sh
+set -e
+
+# If there's a prestart.sh script in the /app directory, run it before starting
+PRE_START_PATH=deploy/docker/prestart.sh
+echo "Checking for script in $PRE_START_PATH"
+if [ -f $PRE_START_PATH ] ; then
+    echo "Running script $PRE_START_PATH"
+    . $PRE_START_PATH
+else
+    echo "There is no script $PRE_START_PATH"
+fi
+
+# Start Supervisor, with Nginx and uWSGI
+echo "Starting server using supervisord..."
+
+exec /usr/bin/supervisord

deploy/docker/supervisord/supervisord-celery_beat.conf

@@ -0,0 +1,12 @@
+[program:celery_beat]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=300
+startinorder=true
+command=/home/mediacms.io/bin/celery beat --pidfile=/home/mediacms.io/mediacms/pids/beat%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_beat.log

deploy/docker/supervisord/supervisord-celery_long.conf

@@ -0,0 +1,13 @@
+[program:celery_long]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=10
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=500
+startinorder=true
+startsecs=0
+command=/home/mediacms.io/bin/celery multi start long1 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_long.log -Ofair --prefetch-multiplier=1 -Q long_tasks

deploy/docker/supervisord/supervisord-celery_short.conf

@@ -0,0 +1,12 @@
+[program:celery_short]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=400
+startinorder=true
+command=/home/mediacms.io/bin/celery multi start short1 short2 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_short.log --soft-time-limit=300 -c10 -Q short_tasks

deploy/docker/supervisord/supervisord-debian.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

deploy/docker/supervisord/supervisord-nginx.conf

@@ -0,0 +1,11 @@
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=200
+startinorder=true
+startsecs=0
+# Graceful stop, see http://nginx.org/en/docs/control.html
+stopsignal=QUIT

deploy/docker/supervisord/supervisord-uwsgi.conf

@@ -0,0 +1,9 @@
+[program:uwsgi]
+command=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/docker/uwsgi.ini
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=100
+startinorder=true
+startsecs=0

deploy/docker/uwsgi.ini

@@ -0,0 +1,24 @@
+[uwsgi]
+
+chdir = /home/mediacms.io/mediacms/
+virtualenv = /home/mediacms.io
+module = cms.wsgi
+
+uid=www-data
+gid=www-data
+
+processes = 2
+threads = 2
+
+master = true
+
+socket = 127.0.0.1:9000
+
+workers = 2
+
+vacuum = true
+
+hook-master-start = unix_signal:15 gracefully_kill_them_all
+need-app = true
+die-on-term = true
+

deploy/local_install/mediacms.io

@@ -59,7 +59,7 @@ server {
     location /media/original {
         alias /home/mediacms.io/mediacms/media_files/original;
         #auth_basic "auth protected area";
-        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/.htpasswd;
+        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/local_install/.htpasswd;
     }
 
     location /media {

deploy/local_install/mediacms.service

@@ -2,7 +2,7 @@
 Description=MediaCMS uwsgi
 
 [Service]
-ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/uwsgi.ini
+ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/local_install/uwsgi.ini
 ExecStop=/usr/bin/killall -9 uwsgi
 RestartSec=3
 #ExecRestart=killall -9 uwsgi; sleep 5; /home/sss/bin/uwsgi --ini /home/sss/wordgames/uwsgi.ini

deploy/local_install/nginx.conf

@@ -0,0 +1,41 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 10240;
+}
+
+    worker_rlimit_nofile    20000;  #each connection needs a filehandle (or 2 if you are proxying)
+http {
+    proxy_connect_timeout 75;
+    proxy_read_timeout 12000;
+    client_max_body_size 5800M;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 10;
+    types_hash_max_size 2048;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers on;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    gzip on;
+    gzip_disable "msie6";
+
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                           '"$request" $status $body_bytes_sent '
+                           '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+    include /etc/nginx/conf.d/*.conf;
+    include /etc/nginx/sites-enabled/*;
+}
+

deploy/local_install/uwsgi.ini

@@ -4,16 +4,24 @@ chdir = /home/mediacms.io/mediacms/
 virtualenv = /home/mediacms.io
 module = cms.wsgi
 
-uid = www-data
-gid = www-data
+uid=www-data
+gid=www-data
+
+processes = 2
+threads = 2
 
-processes = 10
-threads = 10
 master = true
-workers = 8
-vacuum = true
 
 socket = 127.0.0.1:9000
+#socket = /home/mediacms.io/mediacms/deploy/uwsgi.sock
+
+
+workers = 2
+
+
+vacuum = true
 
 logto = /home/mediacms.io/mediacms/logs/errorlog.txt
 
+disable-logging = true
+

deploy/local_install/uwsgi_params

@@ -0,0 +1,16 @@
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;