From af70f34f057a28d27e203125951d3c1cbb314743 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 16:04:03 +0200 Subject: [PATCH 01/15] Remove Vagrant related files --- Procfile.dev | 2 -- Vagrantfile | 15 --------------- bin/vagrant_ctl.sh | 21 --------------------- setup/vagrant/provision.sh | 20 -------------------- 4 files changed, 58 deletions(-) delete mode 100644 Procfile.dev delete mode 100644 Vagrantfile delete mode 100755 bin/vagrant_ctl.sh delete mode 100755 setup/vagrant/provision.sh diff --git a/Procfile.dev b/Procfile.dev deleted file mode 100644 index 289b30418e..0000000000 --- a/Procfile.dev +++ /dev/null @@ -1,2 +0,0 @@ -web: ./manage.py runserver -p $PORT --host 0.0.0.0 -worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 875cca8fa6..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,15 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "redash/dev" - config.vm.synced_folder "./", "/opt/redash/current" - config.vm.network "forwarded_port", guest: 5000, host: 9001 - config.vm.provision "shell" do |s| - s.inline = "/opt/redash/current/setup/vagrant/provision.sh" - s.privileged = false - end -end diff --git a/bin/vagrant_ctl.sh b/bin/vagrant_ctl.sh deleted file mode 100755 index da08abb236..0000000000 --- a/bin/vagrant_ctl.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e - -help() { - echo "Usage: " - echo "`basename "$0"` {start, test}" -} - -case "$1" in - start) - vagrant up - vagrant ssh -c "cd /opt/redash/current; bin/run honcho start -f Procfile.dev;" - ;; - test) - vagrant up - vagrant ssh -c "cd /opt/redash/current; make test" - ;; - *) - help - ;; -esac \ No newline at end of file diff --git a/setup/vagrant/provision.sh b/setup/vagrant/provision.sh deleted file mode 100755 index aad6210c51..0000000000 --- a/setup/vagrant/provision.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - - -cd /opt/redash/current -cp /opt/redash/.env /opt/redash/current -bower install - -#install requirements -sudo pip install -r /opt/redash/current/requirements_dev.txt -sudo pip install -r /opt/redash/current/requirements.txt -sudo pip install pymongo==3.2.1 - -#update database -bin/run ./manage.py database drop_tables -bin/run ./manage.py database create_tables -bin/run ./manage.py users create --admin --password admin "Admin" "admin" - -#Purge Redis cache -redis-cli -n 1 FLUSHALL - From ddf6fc50a580fd71b758450c0081890712ecb1dc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 16:51:24 +0200 Subject: [PATCH 02/15] New Dockerfile and docker-compose for dev --- Dockerfile | 58 +++++---------------- bin/docker-entrypoint | 61 +++++++++++++++++++++++ docker-compose.dev.yml | 50 +++++++++++++++++++ setup/docker/supervisord/supervisord.conf | 56 --------------------- 4 files changed, 123 insertions(+), 102 deletions(-) create mode 100755 bin/docker-entrypoint create mode 100644 docker-compose.dev.yml delete mode 100644 setup/docker/supervisord/supervisord.conf diff --git a/Dockerfile b/Dockerfile index 4824c50398..617a3689b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,17 @@ -FROM ubuntu:trusty +FROM redash/base:latest -# Ubuntu packages -RUN apt-get update && \ - apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget \ - # Postgres client - libpq-dev \ - # Additional packages required for data sources: - libssl-dev libmysqlclient-dev freetds-dev libsasl2-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# Webpack dev server +EXPOSE 8080 -# Users creation -RUN useradd --system --comment " " --create-home redash +# We first copy only the requirements file, to avoid rebuilding on every file +# change. +COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./ +RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt -# Pip requirements for all data source types -RUN pip install -U setuptools==23.1.0 && \ - pip install supervisor==3.1.2 +COPY package.json ./ +RUN npm install -COPY . /opt/redash/current -RUN chown -R redash /opt/redash/current +COPY . ./ +RUN npm run build -# Setting working directory -WORKDIR /opt/redash/current - -# Install project specific dependencies -RUN pip install -r requirements_all_ds.txt && \ - pip install -r requirements.txt - -RUN curl https://deb.nodesource.com/setup_4.x | bash - && \ - apt-get install -y nodejs && \ - sudo -u redash -H make deps && \ - rm -rf node_modules client/node_modules /home/redash/.npm /home/redash/.cache && \ - apt-get purge -y nodejs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Setup supervisord -RUN mkdir -p /opt/redash/supervisord && \ - mkdir -p /opt/redash/logs && \ - cp /opt/redash/current/setup/docker/supervisord/supervisord.conf /opt/redash/supervisord/supervisord.conf - -# Fix permissions -RUN chown -R redash /opt/redash - -# Expose ports -EXPOSE 5000 -EXPOSE 9001 - -# Startup script -CMD ["supervisord", "-c", "/opt/redash/supervisord/supervisord.conf"] +ENTRYPOINT ["/app/bin/docker-entrypoint"] diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint new file mode 100755 index 0000000000..e8cf90f7ca --- /dev/null +++ b/bin/docker-entrypoint @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +worker() { + WORKERS_COUNT=${WORKERS_COUNT:-2} + QUEUES=${QUEUES:-queries,scheduled_queries,celery} + + echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." + exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair +} + +scheduler() { + WORKERS_COUNT=${WORKERS_COUNT:-1} + QUEUES=${QUEUES:-celery} + + echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." + + exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair +} + +api() { + exec sudo -E -u redash /usr/local/bin/gunicorn -b 0.0.0.0:5000 -k gevent --name redash -w5 redash_saas:app +} + +dev_server() { + exec sudo -E -u redash /app/manage.py runserver --debugger --reload -h 0.0.0.0 +} + +create_db() { + exec sudo -E -u redash /app/manage.py database create_tables +} + +help() { + echo "Usage: " + echo "`basename "$0"` {worker, scheduler, api, shell}" +} + +case "$1" in + worker) + shift + worker + ;; + api) + shift + api + ;; + scheduler) + shift + scheduler + ;; + dev_server) + shift + dev_server + ;; + shell) + exec sudo -E -u redash ./manage.py shell + ;; + *) + help + ;; +esac diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000000..f80cb13224 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,50 @@ +version: '2' +services: + api: + build: . + command: dev_server + volumes_from: + - webpack + depends_on: + - webpack + - postgres + - redis + ports: + - "5002:5000" + environment: + PYTHONUNBUFFERED: 0 + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" + worker: + build: . + command: scheduler + volumes_from: + - webpack + depends_on: + - postgres + - redis + - api + environment: + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" + QUEUES: "queries,scheduled_queries,celery" + WORKERS_COUNT: 2 + webpack: + build: . + command: run start + entrypoint: /usr/bin/npm + environment: + DEV_SERVER_HOST: '0.0.0.0' + REDASH_BACKEND: 'http://api:5000' + ports: + - "8080:8080" + volumes: + - ".:/app" + - "/app/client/dist" + - "/app/node_modules" + redis: + image: redis:2.8 + postgres: + image: postgres:9.3 diff --git a/setup/docker/supervisord/supervisord.conf b/setup/docker/supervisord/supervisord.conf deleted file mode 100644 index 9933dfa0fd..0000000000 --- a/setup/docker/supervisord/supervisord.conf +++ /dev/null @@ -1,56 +0,0 @@ -[supervisord] -nodaemon=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -directory=/opt/redash/current - -[inet_http_server] -port = 0.0.0.0:9001 - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[program:redash_server] -command=gunicorn -b 0.0.0.0:5000 --name redash -w 4 --max-requests 1000 redash.wsgi:app -directory=/opt/redash/current -process_name=redash_server -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash - -# There are two queue types here: one for ad-hoc queries, and one for the refresh of scheduled queries -# (note that "scheduled_queries" appears only in the queue list of "redash_celery_scheduled"). -# The default concurrency level for each is 2 (-c2), you can increase based on your machine's resources. -[program:redash_celery] -command=celery worker --app=redash.worker --beat -c2 -Qqueries,celery --maxtasksperchild=10 -Ofair -directory=/opt/redash/current -process_name=redash_celery -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash - -[program:redash_celery_scheduled] -command=celery worker --app=redash.worker -c1 -Qscheduled_queries --maxtasksperchild=10 -Ofair -directory=/opt/redash/current -process_name=redash_celery_scheduled -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash From 06186d9e8d41a45187d60512909ba62c7e29c191 Mon Sep 17 00:00:00 2001 From: Ofer Segev Date: Tue, 17 Jan 2017 20:27:47 +0200 Subject: [PATCH 03/15] Close #242: added ui indication for offline state --- client/app/index.js | 2 +- client/app/services/index.js | 1 + client/app/services/offline-listener.js | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 client/app/services/offline-listener.js diff --git a/client/app/index.js b/client/app/index.js index b6da3b4332..71230fed3e 100644 --- a/client/app/index.js +++ b/client/app/index.js @@ -107,7 +107,7 @@ ngModule.config(($routeProvider, $locationProvider, $compileProvider, }); // Update ui-select's template to use Font-Awesome instead of glyphicon. -ngModule.run(($templateCache) => { +ngModule.run(($templateCache, OfflineListener) => { // eslint-disable-line no-unused-vars const templateName = 'bootstrap/match.tpl.html'; let template = $templateCache.get(templateName); template = template.replace('glyphicon glyphicon-remove', 'fa fa-remove'); diff --git a/client/app/services/index.js b/client/app/services/index.js index 864f8c4e9f..1a00c038df 100644 --- a/client/app/services/index.js +++ b/client/app/services/index.js @@ -12,5 +12,6 @@ export { default as DataSource } from './data-source'; export { default as QuerySnippet } from './query-snippet'; export { default as Notifications } from './notifications'; export { default as KeyboardShortcuts } from './keyboard-shortcuts'; +export { default as OfflineListener } from './offline-listener'; export { default as AlertDialog } from './alert-dialog'; export { default as Auth } from './auth'; diff --git a/client/app/services/offline-listener.js b/client/app/services/offline-listener.js new file mode 100644 index 0000000000..e2d47bd40d --- /dev/null +++ b/client/app/services/offline-listener.js @@ -0,0 +1,23 @@ +function OfflineListener(toastr) { + function addOnlineListener(toast) { + function onlineStateHandler() { + toastr.remove(toast.toastId); + window.removeEventListener('online', onlineStateHandler); + } + window.addEventListener('online', onlineStateHandler); + } + + window.addEventListener('offline', () => { + const toast = toastr.warning('
Please check your Internet connection.
', '', { + allowHtml: true, + autoDismiss: false, + timeOut: false, + tapToDismiss: true, + }); + addOnlineListener(toast); + }); +} + +export default function (ngModule) { + ngModule.service('OfflineListener', OfflineListener); +} From 81fca9329e5b1f9233700205c55d50dcc6a427fa Mon Sep 17 00:00:00 2001 From: Ofer Segev Date: Tue, 17 Jan 2017 21:04:06 +0200 Subject: [PATCH 04/15] App header: query search css fix --- client/app/components/app-header/app-header.css | 7 +++++++ client/app/components/app-header/app-header.html | 6 ++++-- client/app/components/app-header/index.js | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 client/app/components/app-header/app-header.css diff --git a/client/app/components/app-header/app-header.css b/client/app/components/app-header/app-header.css new file mode 100644 index 0000000000..11c7c3fd84 --- /dev/null +++ b/client/app/components/app-header/app-header.css @@ -0,0 +1,7 @@ +.menu-search { + margin-top: 3px; +} + +.menu-search input[type="text"] { + height: 30px; +} \ No newline at end of file diff --git a/client/app/components/app-header/app-header.html b/client/app/components/app-header/app-header.html index 62f9b168e0..aef9bec1af 100644 --- a/client/app/components/app-header/app-header.html +++ b/client/app/components/app-header/app-header.html @@ -37,10 +37,12 @@