Merge pull request #588 from tjwudi/diwu/feature/docker-deployment

Docker deployment support

.dockerignore

@@ -0,0 +1,4 @@
+rd_ui/dist/
+rd_ui/.tmp/
+.git/
+.vagrant/

.env.example

@@ -1,6 +1,6 @@
-export REDASH_STATIC_ASSETS_PATH="../rd_ui/app/"
-export REDASH_LOG_LEVEL="INFO"
-export REDASH_REDIS_URL=redis://localhost:6379/1
-export REDASH_DATABASE_URL="postgresql://redash"
-export REDASH_COOKIE_SECRET=veryverysecret
-export REDASH_GOOGLE_APPS_DOMAIN=
+REDASH_STATIC_ASSETS_PATH="../rd_ui/app/"
+REDASH_LOG_LEVEL="INFO"
+REDASH_REDIS_URL=redis://localhost:6379/1
+REDASH_DATABASE_URL="postgresql://redash"
+REDASH_COOKIE_SECRET=veryverysecret
+REDASH_GOOGLE_APPS_DOMAIN=

.gitignore

@@ -19,3 +19,6 @@ redash/dump.rdb
 venv
 
 dump.rdb
+
+# Docker related
+docker-compose.yaml

Dockerfile

@@ -0,0 +1,47 @@
+FROM ubuntu:trusty
+MAINTAINER Di Wu <diwu@yelp.com>
+
+# Ubuntu packages
+RUN apt-get update && \
+  apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget && \
+  # Postgres client
+  apt-get -y install libpq-dev postgresql-client && \
+  # Additional packages required for data sources:
+  apt-get install -y libssl-dev libmysqlclient-dev
+
+# Users creation
+RUN useradd --system --comment " " --create-home redash
+
+# Pip requirements for all data source types
+RUN pip install -U setuptools && \
+  pip install supervisor==3.1.2
+
+# Download latest source and extract into /opt/redash/current
+# COPY setup/latest_release_url.py /tmp/latest_release_url.py
+# RUN wget $(python /tmp/latest_release_url.py) -O redash.tar.gz && \
+#     mkdir -p /opt/redash/current && \
+#     tar -C /opt/redash/current -xvf redash.tar.gz && \
+#     rm redash.tar.gz
+COPY . /opt/redash/current
+
+# Setting working directory
+WORKDIR /opt/redash/current
+
+# Install project specific dependencies
+RUN pip install -r requirements_all_ds.txt && \
+  pip install -r requirements.txt
+
+# Setup supervisord
+RUN mkdir -p /opt/redash/supervisord && \
+    mkdir -p /opt/redash/logs && \
+    cp /opt/redash/current/setup/files/supervisord_docker.conf /opt/redash/supervisord/supervisord.conf
+
+# Fix permissions
+RUN chown -R redash /opt/redash
+
+# Expose ports
+EXPOSE 5000
+EXPOSE 9001
+
+# Startup script
+CMD ["supervisord", "-c", "/opt/redash/supervisord/supervisord.conf"]

docker-compose-example.yaml

@@ -0,0 +1,22 @@
+redash:
+  image: redash
+  ports:
+    - "5000:5000"
+  links:
+    - redis
+    - postgres
+  env_file: .env
+redis:
+  image: redis:2.8
+postgres:
+  image: postgres:9.3
+  volumes:
+   - /opt/postgres-data:/var/lib/postgresql/data
+redash-nginx:
+  image: redash-nginx:1.0
+  ports:
+    - "80:80"
+  volumes:
+    - "../redash-nginx/nginx.conf:/etc/nginx/nginx.conf"
+  links:
+    - redash

rd_ui/Gruntfile.js

@@ -236,17 +236,6 @@ module.exports = function (grunt) {
     //   dist: {}
     // },
 
-    imagemin: {
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.app %>/images',
-          src: '{,*/}*.{png,jpg,jpeg,gif}',
-          dest: '<%= yeoman.dist %>/images'
-        }]
-      }
-    },
-
     svgmin: {
       dist: {
         files: [{
@@ -348,7 +337,6 @@ module.exports = function (grunt) {
       ],
       dist: [
         'copy:styles',
-        'imagemin',
         'svgmin'
       ]
     },

rd_ui/package.json

@@ -12,7 +12,6 @@
     "grunt-contrib-copy": "^0.5.0",
     "grunt-contrib-cssmin": "^0.9.0",
     "grunt-contrib-htmlmin": "^0.3.0",
-    "grunt-contrib-imagemin": "^0.7.0",
     "grunt-contrib-jshint": "^0.10.0",
     "grunt-contrib-uglify": "^0.4.0",
     "grunt-contrib-watch": "^0.6.1",

setup/docker_init_postgres.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Create database / tables
+pg_user_exists=0
+psql --host=postgres --username=postgres postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash postgres user & database."
+     createuser redash --username=postgres --host=postgres --no-superuser --no-createdb --no-createrole
+     createdb redash --username=postgres --host=postgres --owner=redash
+
+    cd /opt/redash/current
+    ./manage.py database create_tables
+fi
+
+# Create default admin user
+cd /opt/redash/current
+# TODO: make sure user created only once
+# TODO: generate temp password and print to screen
+./manage.py users create --admin --password admin "Admin" "admin"
+
+# Create re:dash read only pg user & setup data source
+pg_user_exists=0
+psql --host=postgres --username=postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash_reader'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash reader postgres user."
+    REDASH_READER_PASSWORD=$(pwgen -1)
+    psql --host=postgres --username=postgres -c "CREATE ROLE redash_reader WITH PASSWORD '$REDASH_READER_PASSWORD' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN"
+    psql --host=postgres --username=postgres -c "grant select(id,name,type) ON data_sources to redash_reader;" redash
+    psql --host=postgres --username=postgres -c "grant select(id,name) ON users to redash_reader;" redash
+    psql --host=postgres --username=postgres -c "grant select on activity_log, events, queries, dashboards, widgets, visualizations, query_results to redash_reader;" redash
+
+    cd /opt/redash/current
+    ./manage.py ds new -n "re:dash metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"$REDASH_READER_PASSWORD\", \"host\": \"localhost\", \"dbname\": \"redash\"}"
+fi

setup/files/docker-redash-nginx/Dockerfile

@@ -0,0 +1,8 @@
+FROM nginx
+MAINTAINER Di Wu <diwu@yelp.com>
+
+COPY nginx.conf /etc/nginx/nginx.conf
+
+RUN mkdir -p /var/log/nginx/log && \
+  touch /var/log/nginx/log/access.log && \
+  touch /var/log/nginx/log/error.log

setup/files/docker-redash-nginx/nginx.conf

@@ -0,0 +1,28 @@
+events {
+  worker_connections  4096;  ## Default: 1024
+}
+
+http {
+  upstream redashapp {
+    server redash:5000;
+  }
+
+  server {
+    listen  80;
+
+    access_log  /var/log/nginx/log/access.log;
+    error_log  /var/log/nginx/log/error.log;
+
+    gzip on;
+    gzip_types *;
+    gzip_proxied any;
+
+    location / {
+      proxy_pass  http://redashapp;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+  }
+}

setup/files/supervisord.conf

@@ -33,7 +33,7 @@ autostart=true
 autorestart=true
 stdout_logfile=/opt/redash/logs/celery.log
 stderr_logfile=/opt/redash/logs/celery_error.log
- 
+
 [program:redash_celery_scheduled]
 command=/opt/redash/current/bin/run celery worker --app=redash.worker -c2 -Qscheduled_queries
 process_name=redash_celery_scheduled

setup/files/supervisord_docker.conf

@@ -0,0 +1,48 @@
+[supervisord]
+nodaemon=true
+logfile=/opt/redash/logs/supervisord.log
+pidfile=/opt/redash/supervisord/supervisord.pid
+directory=/opt/redash/current
+
+[inet_http_server]
+port = 0.0.0.0:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:redash_server]
+command=/opt/redash/current/bin/run gunicorn -b 0.0.0.0:5000 --name redash -w 4 redash.wsgi:app
+directory=/opt/redash/current
+process_name=redash_server
+numprocs=1
+priority=999
+autostart=true
+autorestart=true
+stdout_logfile=/opt/redash/logs/api.log
+stderr_logfile=/opt/redash/logs/api_error.log
+
+# There are two queue types here: one for ad-hoc queries, and one for the refresh of scheduled queries
+# (note that "scheduled_queries" appears only in the queue list of "redash_celery_scheduled").
+# The default concurrency level for each is 2 (-c2), you can increase based on your machine's resources.
+
+[program:redash_celery]
+command=sudo -u redash /opt/redash/current/bin/run celery worker --app=redash.worker --beat -c2 -Qqueries,celery
+directory=/opt/redash/current
+process_name=redash_celery
+numprocs=1
+priority=999
+autostart=true
+autorestart=true
+stdout_logfile=/opt/redash/logs/celery.log
+stderr_logfile=/opt/redash/logs/celery_error.log
+
+[program:redash_celery_scheduled]
+command=sudo -u redash /opt/redash/current/bin/run celery worker --app=redash.worker -c2 -Qscheduled_queries
+directory=/opt/redash/current
+process_name=redash_celery_scheduled
+numprocs=1
+priority=999
+autostart=true
+autorestart=true
+stdout_logfile=/opt/redash/logs/celery.log
+stderr_logfile=/opt/redash/logs/celery_error.log

setup/latest_release_url.py

@@ -0,0 +1,6 @@
+import urllib2
+import json
+
+latest = json.load(urllib2.urlopen("https://api.github.com/repos/EverythingMe/redash/releases/latest"))
+
+print latest['assets'][0]['browser_download_url']