From 1033be207ca05dbc511ae6c7596bb12e438ed410 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Fri, 18 Nov 2016 15:48:01 +0300 Subject: [PATCH 1/5] clickhouse: query runner for Yandex.Clickhouse DB --- query_runner/clickhouse.py | 119 +++++++++++++++++++++++++++++++++++++ settings.py | 1 + 2 files changed, 120 insertions(+) create mode 100644 query_runner/clickhouse.py diff --git a/query_runner/clickhouse.py b/query_runner/clickhouse.py new file mode 100644 index 0000000000..07c3156344 --- /dev/null +++ b/query_runner/clickhouse.py @@ -0,0 +1,119 @@ +import json +import logging +from redash.query_runner import * +from redash.utils import JSONEncoder +import pprint +logger = logging.getLogger(__name__) + +pp = pprint.PrettyPrinter() + +try: + import requests + enabled = True +except ImportError as e: + logger.info(str(e)) + enabled = False + + +class ClickHouse(BaseSQLQueryRunner): + noop_query = "SELECT 1" + + @classmethod + def configuration_schema(cls): + return { + "type": "object", + "properties": { + "user": { + "type": "string", + "default": "default" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 8123 + }, + "dbname": { + "type": "string", + "title": "Database Name" + } + }, + "required": ["dbname"], + "secret": ["password"] + } + + @classmethod + def type(cls): + return "clickhouse" + + def __init__(self, configuration): + super(ClickHouse, self).__init__(configuration) + + def _get_tables(self, schema): + query = "SELECT database, table, name FROM system.columns WHERE database NOT IN ('system')" + + results, error = self.run_query(query, None) + + if error is not None: + raise Exception("Failed getting schema.") + + results = json.loads(results) + + for row in results['rows']: + table_name = '{}.{}'.format(row['database'], row['table']) + + if table_name not in schema: + schema[table_name] = {'name': table_name, 'columns': []} + + schema[table_name]['columns'].append(row['name']) + + return schema.values() + + def _send_query(self, data, stream=False): + url = 'http://{host}:{port}'.format(host=self.configuration['host'], port=self.configuration['port']) + r = requests.post(url, data=data, stream=stream, params={ + 'user': self.configuration['user'], 'password': self.configuration['password'], + 'database': self.configuration['dbname'] + }) + if r.status_code != 200: + raise Exception(r.text) + return r.json() + + @staticmethod + def _define_column_type(column): + c = column.lower() + if 'int' in c: + return 'integer' + elif 'float' in c: + return 'float' + else: + return 'string' + + def _clickhouse_query(self, query): + query += ' FORMAT JSON' + result = self._send_query(query) + columns = [{'name': r['name'], 'friendly_name': r['name'], + 'type': self._define_column_type(r['type'])} for r in result['meta']] + return {'columns': columns, 'rows': result['data']} + + def run_query(self, query, user): + logger.info("Clickhouse is about to execute query: %s", query) + if query == "": + json_data = None + error = "Query is empty" + return json_data, error + try: + q = self._clickhouse_query(query) + data = json.dumps(q, cls=JSONEncoder) + error = None + except Exception as exc: + data = None + error = str(exc) + return data, error + +register(ClickHouse) diff --git a/settings.py b/settings.py index f597c9495a..2e69964174 100644 --- a/settings.py +++ b/settings.py @@ -169,6 +169,7 @@ def all_settings(): 'redash.query_runner.hive_ds', 'redash.query_runner.impala_ds', 'redash.query_runner.vertica', + 'redash.query_runner.clickhouse', 'redash.query_runner.treasuredata', 'redash.query_runner.sqlite', 'redash.query_runner.dynamodb_sql', From 718feffd20d098521c9ee4daa604fa5b00f14215 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Sun, 20 Nov 2016 13:14:17 +0300 Subject: [PATCH 2/5] clickhouse: added more column types --- query_runner/clickhouse.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/query_runner/clickhouse.py b/query_runner/clickhouse.py index 07c3156344..f7d8afc08b 100644 --- a/query_runner/clickhouse.py +++ b/query_runner/clickhouse.py @@ -88,11 +88,15 @@ def _send_query(self, data, stream=False): def _define_column_type(column): c = column.lower() if 'int' in c: - return 'integer' + return TYPE_INTEGER elif 'float' in c: - return 'float' + return TYPE_FLOAT + elif 'datetime' == c: + return TYPE_DATETIME + elif 'date' == c: + return TYPE_DATE else: - return 'string' + return TYPE_STRING def _clickhouse_query(self, query): query += ' FORMAT JSON' From 04198a1f7576f003bc90159f41844ccd9e489774 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Sun, 20 Nov 2016 14:50:38 +0300 Subject: [PATCH 3/5] clickhouse: removed unused code --- query_runner/clickhouse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/query_runner/clickhouse.py b/query_runner/clickhouse.py index f7d8afc08b..a6f29199eb 100644 --- a/query_runner/clickhouse.py +++ b/query_runner/clickhouse.py @@ -2,11 +2,8 @@ import logging from redash.query_runner import * from redash.utils import JSONEncoder -import pprint logger = logging.getLogger(__name__) -pp = pprint.PrettyPrinter() - try: import requests enabled = True From 90b8ab5784b8dfac67bc28abb0cec77dfd43e92b Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Wed, 23 Nov 2016 10:15:03 +0300 Subject: [PATCH 4/5] review fixes --- query_runner/clickhouse.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/query_runner/clickhouse.py b/query_runner/clickhouse.py index a6f29199eb..e071522ecc 100644 --- a/query_runner/clickhouse.py +++ b/query_runner/clickhouse.py @@ -2,15 +2,9 @@ import logging from redash.query_runner import * from redash.utils import JSONEncoder +import requests logger = logging.getLogger(__name__) -try: - import requests - enabled = True -except ImportError as e: - logger.info(str(e)) - enabled = False - class ClickHouse(BaseSQLQueryRunner): noop_query = "SELECT 1" @@ -20,6 +14,10 @@ def configuration_schema(cls): return { "type": "object", "properties": { + "url": { + "type": "string", + "default": "http://127.0.0.1:8123" + }, "user": { "type": "string", "default": "default" @@ -27,14 +25,6 @@ def configuration_schema(cls): "password": { "type": "string" }, - "host": { - "type": "string", - "default": "127.0.0.1" - }, - "port": { - "type": "number", - "default": 8123 - }, "dbname": { "type": "string", "title": "Database Name" @@ -72,8 +62,7 @@ def _get_tables(self, schema): return schema.values() def _send_query(self, data, stream=False): - url = 'http://{host}:{port}'.format(host=self.configuration['host'], port=self.configuration['port']) - r = requests.post(url, data=data, stream=stream, params={ + r = requests.post(self.configuration['url'], data=data, stream=stream, params={ 'user': self.configuration['user'], 'password': self.configuration['password'], 'database': self.configuration['dbname'] }) @@ -103,7 +92,7 @@ def _clickhouse_query(self, query): return {'columns': columns, 'rows': result['data']} def run_query(self, query, user): - logger.info("Clickhouse is about to execute query: %s", query) + logger.debug("Clickhouse is about to execute query: %s", query) if query == "": json_data = None error = "Query is empty" @@ -112,9 +101,10 @@ def run_query(self, query, user): q = self._clickhouse_query(query) data = json.dumps(q, cls=JSONEncoder) error = None - except Exception as exc: + except Exception as e: data = None - error = str(exc) + logging.exception(e) + error = str(e) return data, error register(ClickHouse) From 0afcac8ac2a8a239b9776cd14340a6eea3c05b9d Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Wed, 23 Nov 2016 11:53:11 +0300 Subject: [PATCH 5/5] clickhouse: changed exception's message type to unicode --- query_runner/clickhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query_runner/clickhouse.py b/query_runner/clickhouse.py index e071522ecc..9574a6f202 100644 --- a/query_runner/clickhouse.py +++ b/query_runner/clickhouse.py @@ -104,7 +104,7 @@ def run_query(self, query, user): except Exception as e: data = None logging.exception(e) - error = str(e) + error = unicode(e) return data, error register(ClickHouse)