Simple db init with docker

.dockerignore

@@ -1,4 +1,4 @@
-rd_ui/dist/
 rd_ui/.tmp/
+rd_ui/node_modules/
 .git/
 .vagrant/

.gitignore

@@ -21,4 +21,4 @@ venv
 dump.rdb
 
 # Docker related
-docker-compose.yaml
+docker-compose.yml

Dockerfile

@@ -3,11 +3,11 @@ MAINTAINER Di Wu <diwu@yelp.com>
 
 # Ubuntu packages
 RUN apt-get update && \
-  apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget && \
+  apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget \
   # Postgres client
-  apt-get -y install libpq-dev postgresql-client && \
+  libpq-dev \
   # Additional packages required for data sources:
-  apt-get install -y libssl-dev libmysqlclient-dev
+  libssl-dev libmysqlclient-dev
 
 # Users creation
 RUN useradd --system --comment " " --create-home redash
@@ -16,12 +16,6 @@ RUN useradd --system --comment " " --create-home redash
 RUN pip install -U setuptools && \
   pip install supervisor==3.1.2
 
-# Download latest source and extract into /opt/redash/current
-# COPY setup/latest_release_url.py /tmp/latest_release_url.py
-# RUN wget $(python /tmp/latest_release_url.py) -O redash.tar.gz && \
-#     mkdir -p /opt/redash/current && \
-#     tar -C /opt/redash/current -xvf redash.tar.gz && \
-#     rm redash.tar.gz
 COPY . /opt/redash/current
 
 # Setting working directory
@@ -34,11 +28,14 @@ RUN pip install -r requirements_all_ds.txt && \
 # Setup supervisord
 RUN mkdir -p /opt/redash/supervisord && \
     mkdir -p /opt/redash/logs && \
-    cp /opt/redash/current/setup/files/supervisord_docker.conf /opt/redash/supervisord/supervisord.conf
+    cp /opt/redash/current/setup/docker/supervisord/supervisord.conf /opt/redash/supervisord/supervisord.conf
 
 # Fix permissions
 RUN chown -R redash /opt/redash
 
+# Expose init script as /init.sh
+RUN ln -s /opt/redash/current/setup/docker/redash_database_init.sh /init.sh
+
 # Expose ports
 EXPOSE 5000
 EXPOSE 9001

circle.yml

@@ -1,15 +1,14 @@
 machine:
+  services:
+    - docker
   node:
     version:
-      0.10.24
+      0.12.4
   python:
     version:
       2.7.3
 dependencies:
   pre:
-    - wget http://downloads.sourceforge.net/project/optipng/OptiPNG/optipng-0.7.5/optipng-0.7.5.tar.gz
-    - tar xvf optipng-0.7.5.tar.gz
-    - cd optipng-0.7.5; ./configure; make; sudo checkinstall -y;
     - make deps
     - pip install -r requirements_dev.txt
     - pip install -r requirements.txt
@@ -26,6 +25,13 @@ deployment:
     branch: master
     commands:
       - make upload
+  docker:
+    branch: [master, docker]
+    commands:
+        - echo "rd_ui/app" >> .dockerignore
+        - docker build -t everythingme/redash:$(./manage.py version | sed -e "s/\+/./") .
+        - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+        - docker push everythingme/redash:$(./manage.py version | sed -e "s/\+/./")
 notify:
   webhooks:
     - url: https://webhooks.gitter.im/e/895d09c3165a0913ac2f

redash/settings.py

@@ -19,6 +19,23 @@ def parse_db_url(url):
 
     return connection
 
+def parse_db_args(name, host, port, user, password, url):
+    if url is not None:
+        return parse_db_url(url)
+    connection = {'threadlocals': True}
+
+    if name is not None:
+        connection['name'] = name
+    if name is not None:
+        connection['host'] = host
+    if name is not None:
+        connection['port'] = port
+    if name is not None:
+        connection['user'] = user
+    if name is not None:
+        connection['password'] = password
+
+    return connection
 
 def fix_assets_path(path):
     fullpath = os.path.join(os.path.dirname(__file__), path)
@@ -61,7 +78,14 @@ def all_settings():
 STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)
-DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres"))
+# Use either NAME+HOST+PORT+USER+PASSWORD or URL. URL override other values
+DATABASE_NAME = os.environ.get("REDASH_DATABASE_NAME", "postgres")
+DATABASE_HOST = os.environ.get("REDASH_DATABASE_HOST", None)
+DATABASE_PORT = os.environ.get("REDASH_DATABASE_PORT", None)
+DATABASE_USER = os.environ.get("REDASH_DATABASE_USER", None)
+DATABASE_PASSWORD = os.environ.get("REDASH_DATABASE_PASSWORD", None)
+DATABASE_URL = os.environ.get("REDASH_DATABASE_URL", None)
+DATABASE_CONFIG = parse_db_args(DATABASE_NAME, DATABASE_HOST, DATABASE_PORT, DATABASE_USER, DATABASE_PASSWORD, DATABASE_URL)
 
 # Celery related settings
 CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)

setup/amazon_linux/README.md

@@ -0,0 +1 @@
+Bootstrap script for Amazon Linux AMI. *Not supported*, we recommend to use the Docker images instead.

setup/bootstrap_amazon_linux.sh → setup/amazon_linux/bootstrap.sh

@@ -2,8 +2,7 @@
 set -eu
 
 REDASH_BASE_PATH=/opt/redash
-FILES_BASE_URL=https://github.com/raw/EverythingMe/redash/docs_setup/setup/files/
-FILE_BASE_URL_FOR_AMAZON_LINUX=https://github.com/raw/EverythingMe/redash/master/setup/files/
+FILES_BASE_URL=https://github.com/raw/EverythingMe/redash/master/setup/amazon_linux/files/
 # Verify running as root:
 if [ "$(id -u)" != "0" ]; then
     if [ $# -ne 0 ]; then
@@ -178,7 +177,7 @@ fi
 
 
 # Get supervisord startup script
-sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILE_BASE_URL_FOR_AMAZON_LINUX"supervisord_for_amazon_linux.conf"
+sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord.conf"
 
 # install start-stop-daemon
 wget http://developer.axis.com/download/distribution/apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz
@@ -187,7 +186,7 @@ cd apps/sys-utils/start-stop-daemon-IR1_9_18-2/
 gcc start-stop-daemon.c -o start-stop-daemon
 cp start-stop-daemon /sbin/
 
-wget -O /etc/init.d/redash_supervisord $FILE_BASE_URL_FOR_AMAZON_LINUX"redash_supervisord_init_for_amazon_linux"
+wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init"
 add_service "redash_supervisord"
 
 # Nginx setup

setup/docker/README.md

@@ -0,0 +1 @@
+Files used for the Docker image creation.

setup/docker/create_database.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# This script assumes you're using docker-compose, with at least two images: redash for the redash instance
+# and postgres for the postgres instance.
+#
+# This script is not idempotent and should be run once.
+
+run_redash="docker-compose run --rm redash"
+
+$run_redash /opt/redash/current/manage.py database create_tables
+
+# Create default admin user
+$run_redash /opt/redash/current/manage.py users create --admin --password admin "Admin" "admin"
+
+# This is a hack to get the Postgres IP and PORT from the instance itself.
+temp_env_file=`mktemp /tmp/pg_env.XXXXXX`
+docker-compose run --rm postgres env > $temp_env_file
+source $temp_env_file
+
+run_psql="docker-compose run --rm postgres psql -h $POSTGRES_PORT_5432_TCP_ADDR -p $POSTGRES_PORT_5432_TCP_PORT -U postgres"
+
+# Create redash_reader user. We don't use a strong password, as the instance supposed to be accesible only from the redash host.
+$run_psql -c "CREATE ROLE redash_reader WITH PASSWORD 'redash_reader' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN"
+$run_psql -c "grant select(id,name,type) ON data_sources to redash_reader;"
+$run_psql -c "grant select(id,name) ON users to redash_reader;"
+$run_psql -c "grant select on activity_log, events, queries, dashboards, widgets, visualizations, query_results to redash_reader;"
+
+$run_redash /opt/redash/current/manage.py ds new -n "re:dash metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"redash_reader\", \"host\": \"postgres\", \"dbname\": \"postgres\"}"

setup/docker/env

@@ -0,0 +1,6 @@
+REDASH_STATIC_ASSETS_PATH=../rd_ui/dist/
+REDASH_LOG_LEVEL=INFO
+REDASH_REDIS_URL=redis://redis:6379/0
+REDASH_DATABASE_URL=postgresql://postgres@postgres/postgres
+REDASH_COOKIE_SECRET=veryverysecret
+REDASH_GOOGLE_APPS_DOMAIN=

setup/docker/redash_database_init.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+import os
+import sys
+import psycopg2
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+
+DATABASE_NAME = os.environ.get("REDASH_DATABASE_NAME", "postgres")
+DATABASE_HOST = os.environ.get("REDASH_DATABASE_HOST", None)
+DATABASE_PORT = os.environ.get("REDASH_DATABASE_PORT", None)
+DATABASE_USER = os.environ.get("REDASH_DATABASE_USER", None)
+DATABASE_PASSWORD = os.environ.get("REDASH_DATABASE_PASSWORD", None)
+DATABASE_USER_ADMIN = os.environ.get("REDASH_DATABASE_USER_ADMIN", None)
+DATABASE_PASSWORD_ADMIN = os.environ.get("REDASH_DATABASE_PASSWORD_ADMIN", None)
+
+connection = {}
+connection_admin = {}
+
+if DATABASE_NAME is not None:
+    connection['database'] = DATABASE_NAME
+if DATABASE_HOST is not None:
+    connection['host'] = DATABASE_HOST
+    connection_admin['host'] = DATABASE_HOST
+if DATABASE_PORT is not None:
+    connection['port'] = DATABASE_PORT
+    connection_admin['port'] = DATABASE_PORT
+if DATABASE_USER is not None:
+    connection['user'] = DATABASE_USER
+if DATABASE_PASSWORD is not None:
+    connection['password'] = DATABASE_PASSWORD
+if DATABASE_USER_ADMIN is not None:
+    connection_admin['user'] = DATABASE_USER_ADMIN
+if DATABASE_PASSWORD_ADMIN is not None:
+    connection_admin['password'] = DATABASE_PASSWORD_ADMIN
+
+def output(text):
+    os.stdout(text+'\n')
+
+def error(text):
+    os.stderr(text+'\n')
+
+def fatal(text):
+    error(text)
+    sys.exit(1)
+
+if sys.argv is None:
+    fatal("No argv exists, you're doomed. Python doesn't work as expected !")
+
+if len(sys.argv)<=1:
+    fatal("You must provide one argument (either )")
+
+if DATABASE_NAME is None or DATABASE_NAME == '':
+    fatal("You must provide a non null database name")
+
+command = sys.argv[1]
+
+if command == 'create_db_and_role':
+    conn = psycopg2.connect(**connection_admin)
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = conn.cursor()
+    cursor.execute("select * from pg_database where datname = %(database_name)s", { 'database_name': DATABASE_NAME })
+    databases = cursor.fetchall()
+    database_created = False
+    if len(databases) == 0:
+        cursor.execute("CREATE DATABASE %(database_name)s" % { 'database_name': DATABASE_NAME })
+        database_created = True
+    if DATABASE_USER is not None:
+        cursor.execute("select 1 from pg_roles where rolname = %(role_name)s", { 'role_name': DATABASE_USER })
+        role_created = False
+        roles = cursor.fetchall()
+        if len(roles) == 0:
+            if DATABASE_PASSWORD is None:
+                cursor.execute("CREATE ROLE %(role_name)s WITH LOGIN CREATEDB" % { 'role_name': DATABASE_USER })
+            else:
+                cursor.execute("CREATE ROLE %(role_name)s WITH LOGIN CREATEDB PASSWORD '%(role_password)s'" % { 'role_name': DATABASE_USER, 'role_password': DATABASE_PASSWORD })
+            role_created = True
+        if database_created or role_created:
+            cursor.execute("GRANT ALL PRIVILEGES ON DATABASE %(database_name)s to %(role_name)s" % { 'role_name': DATABASE_USER, 'database_name': DATABASE_NAME })
+
+elif command == 'check_database_init':
+    conn = psycopg2.connect(**connection)
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = conn.cursor()
+    cursor.execute("select * from information_schema.tables where table_name='dashboards'")
+    tables = cursor.fetchall()
+    if len(tables) == 0:
+        sys.exit(2)
+
+elif command == 'create_reader_role':
+    conn = psycopg2.connect(**connection)
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = conn.cursor()
+    cursor.execute("select 1 from pg_roles where rolname = 'redash_reader'")
+    if len(cursor.fetchall()) == 0:
+        cursor.execute("CREATE ROLE redash_reader WITH PASSWORD 'redash_reader' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN")
+    cursor.execute("grant select(id,name,type) ON data_sources to redash_reader")
+    cursor.execute("grant select(id,name) ON users to redash_reader")
+
+elif command == 'check_redash_metadata':
+    conn = psycopg2.connect(**connection)
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = conn.cursor()
+    cursor.execute("select 1 from data_sources where name = %(name)s", { 'name': 're:dash metadata' })
+    if len(cursor.fetchall()) == 0:
+        sys.exit(2)
+
+else:
+    fatal("The argument [%s] isn't recognized as valid parameter for this script" % (command,))
+

setup/docker/redash_database_init.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+workpath="/opt/redash/current"
+manage="${workpath}/manage.py"
+database_init="${workpath}/setup/docker/redash_database_init.py"
+
+[ -z "${REDASH_DEFAULT_ADMIN_NAME}" ] && REDASH_DEFAULT_ADMIN_NAME="Admin"
+[ -z "${REDASH_DEFAULT_ADMIN_LOGIN}" ] && REDASH_DEFAULT_ADMIN_LOGIN="admin"
+[ -z "${REDASH_DEFAULT_ADMIN_PASSWORD}" ] && REDASH_DEFAULT_ADMIN_PASSWORD="admin"
+
+# [ -z "${REDASH_DATABASE_HOST}" ]
+[ -z "${REDASH_DATABASE_NAME}" ] && REDASH_DATABASE_NAME="postgres"
+
+final() {
+    exit $1
+}
+
+fail() {
+    echo "$1" >&2
+    final 1
+}
+
+end() {
+    final 0
+}
+
+"${database_init}" "create_db_and_role" || fail "Something went wrong during database and role creation"
+
+if "${database_init}" "check_database_init"
+then
+    echo "database seems to be already populated... doing nothing"
+else
+    echo "database seems to be empty. Creating tables"
+    "${manage}" database create_tables
+    "${manage}" users create --admin --password "${REDASH_DEFAULT_ADMIN_PASSWORD}" "${REDASH_DEFAULT_ADMIN_NAME}" "${REDASH_DEFAULT_ADMIN_LOGIN}"
+fi
+
+"${database_init}" "create_reader_role"
+
+if "${database_init}" "check_redash_metadata"
+then
+    echo "re:dash metadata seems to already exists... doing nothing"
+else
+    echo "re:dash metadata seems to not be present. Inserting enreg"
+    "${manage}" ds new -n "re:dash metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"redash_reader\", \"host\": \"${REDASH_DATABASE_HOST}\", \"dbname\": \"${REDASH_DATABASE_NAME}\"}"
+fi
+