From 31f527fefe2f7807121d35d60fdce76b58ec0a81 Mon Sep 17 00:00:00 2001 From: Rory Miller Date: Wed, 30 Sep 2020 01:33:10 +0100 Subject: [PATCH] Superset additions initial commit dtable, email func --- requirements.txt | 2 + superset/app.py | 19 +- superset/common/routes.py | 16 + superset/dtable/__init__.py | 16 + superset/dtable/common/__init__.py | 0 superset/dtable/common/routes.py | 16 + superset/dtable/config.py | 4 + superset/dtable/css/jquery.css | 144 +++++++ superset/dtable/div.dataTables_sizing | 0 superset/dtable/div.dataTables_sizing, | 0 superset/dtable/jquery.css | 144 +++++++ superset/dtable/mod_tables/__init__.py | 0 superset/dtable/mod_tables/controllers.py | 59 +++ superset/dtable/mod_tables/models.py | 96 +++++ .../dtable/mod_tables/serverside/__init__.py | 0 .../mod_tables/serverside/serverside_table.py | 261 +++++++++++++ superset/dtable/static/js/get_columns.py | 11 + superset/dtable/static/js/serverside_table.js | 69 ++++ superset/dtable/templates/css/jquery.css | 1 + .../dtable/templates/serverside_table.html | 31 ++ superset/dtable/templates/template.html | 27 ++ superset/models/schedules.py | 52 ++- superset/models/slice.py | 13 + superset/tasks/schedules.py | 359 ++++++------------ superset/templates/superset/add_dtable.html | 32 ++ superset/translations/messages.pot | 44 --- superset/views/chart/mixin.py | 3 +- superset/views/chart/views.py | 6 +- superset/views/core.py | 86 ++++- superset/views/dtable.py | 28 ++ superset/views/schedules.py | 125 ++++-- 31 files changed, 1303 insertions(+), 361 deletions(-) create mode 100644 superset/common/routes.py create mode 100644 superset/dtable/__init__.py create mode 100644 superset/dtable/common/__init__.py create mode 100644 superset/dtable/common/routes.py create mode 100644 superset/dtable/config.py create mode 100644 superset/dtable/css/jquery.css create mode 100644 superset/dtable/div.dataTables_sizing create mode 100644 superset/dtable/div.dataTables_sizing, create mode 100644 superset/dtable/jquery.css create mode 100644 superset/dtable/mod_tables/__init__.py create mode 100644 superset/dtable/mod_tables/controllers.py create mode 100644 superset/dtable/mod_tables/models.py create mode 100644 superset/dtable/mod_tables/serverside/__init__.py create mode 100644 superset/dtable/mod_tables/serverside/serverside_table.py create mode 100644 superset/dtable/static/js/get_columns.py create mode 100644 superset/dtable/static/js/serverside_table.js create mode 100644 superset/dtable/templates/css/jquery.css create mode 100644 superset/dtable/templates/serverside_table.html create mode 100644 superset/dtable/templates/template.html create mode 100644 superset/templates/superset/add_dtable.html create mode 100644 superset/views/dtable.py diff --git a/requirements.txt b/requirements.txt index 1ed3e7b721..95373707c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -101,3 +101,5 @@ zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools + +beautifulsoup4==4.9.1 diff --git a/superset/app.py b/superset/app.py index 7dc7aec5f4..50db82c3e2 100644 --- a/superset/app.py +++ b/superset/app.py @@ -45,6 +45,9 @@ from superset.typing import FlaskResponse from superset.utils.core import pessimistic_connection_handling from superset.utils.log import DBEventLogger, get_event_logger_from_cfg_value +from superset.dtable.mod_tables.models import TableBuilder +from superset.dtable.mod_tables.controllers import tables + logger = logging.getLogger(__name__) @@ -59,7 +62,7 @@ def create_app() -> Flask: app_initializer = app.config.get("APP_INITIALIZER", SupersetAppInitializer)(app) app_initializer.init_app() - + app.register_blueprint(tables) return app # Make sure that bootstrap errors ALWAYS get logged @@ -144,7 +147,7 @@ def init_views(self) -> None: AnnotationModelView, ) from superset.views.api import Api - from superset.views.core import Superset + from superset.views.core import Superset, DataTableView from superset.views.redirects import R from superset.views.key_value import KV from superset.views.access_requests import AccessRequestsModelView @@ -166,6 +169,7 @@ def init_views(self) -> None: from superset.views.schedules import ( DashboardEmailScheduleView, SliceEmailScheduleView, + S3ScheduleView, ) from superset.views.sql_lab import ( QueryView, @@ -241,6 +245,7 @@ def init_views(self) -> None: category_label=__("Manage"), category_icon="", ) + appbuilder.add_view( QueryView, "Queries", @@ -266,9 +271,9 @@ def init_views(self) -> None: appbuilder.add_view_no_menu(CssTemplateAsyncModelView) appbuilder.add_view_no_menu(CsvToDatabaseView) appbuilder.add_view_no_menu(Dashboard) + appbuilder.add_view_no_menu(DataTableView()) appbuilder.add_view_no_menu(DashboardModelViewAsync) appbuilder.add_view_no_menu(Datasource) - if feature_flag_manager.is_feature_enabled("KV_STORE"): appbuilder.add_view_no_menu(KV) @@ -369,6 +374,14 @@ def init_views(self) -> None: category_label=__("Manage"), icon="fa-search", ) + appbuilder.add_view( + S3ScheduleView, + "S3 Schedule", + label=__("S3 Export Schedules"), + category="Manage", + category_label=__("Manage"), + icon="fa-search", + ) # # Conditionally add Access Request Model View diff --git a/superset/common/routes.py b/superset/common/routes.py new file mode 100644 index 0000000000..505fafbb88 --- /dev/null +++ b/superset/common/routes.py @@ -0,0 +1,16 @@ +from flask import Blueprint, render_template + +main = Blueprint('main', __name__, url_prefix='') + + +@main.route("/") +def index(): + return render_template("index.html") + +@main.route("/clientside_table") +def clientside_table(): + return render_template("clientside_table.html") + +@main.route("/serverside_table") +def serverside_table(): + return render_template("serverside_table.html") diff --git a/superset/dtable/__init__.py b/superset/dtable/__init__.py new file mode 100644 index 0000000000..605838e45d --- /dev/null +++ b/superset/dtable/__init__.py @@ -0,0 +1,16 @@ +from flask import Flask, redirect, session +from superset.dtable.mod_tables.models import TableBuilder + + +flask_app = Flask(__name__) + +table_builder = TableBuilder() + + +from superset.dtable.common.routes import main +from superset.dtable.mod_tables.controllers import tables + + +# Register the different blueprints +flask_app.register_blueprint(main) +flask_app.register_blueprint(tables) diff --git a/superset/dtable/common/__init__.py b/superset/dtable/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/superset/dtable/common/routes.py b/superset/dtable/common/routes.py new file mode 100644 index 0000000000..9a6106721f --- /dev/null +++ b/superset/dtable/common/routes.py @@ -0,0 +1,16 @@ +from flask import Blueprint, render_template + +main = Blueprint('main', __name__, url_prefix='/dtable') + + +@main.route("/") +def index(): + return render_template("index.html") + +@main.route("/clientside_table") +def clientside_table(): + return render_template("clientside_table.html") + +@main.route("/serverside_table") +def serverside_table(): + return render_template("serverside_table.html") diff --git a/superset/dtable/config.py b/superset/dtable/config.py new file mode 100644 index 0000000000..f1f1474c98 --- /dev/null +++ b/superset/dtable/config.py @@ -0,0 +1,4 @@ +FLASK_ENV=development +debug=True +DEBUG=True +Debug=True diff --git a/superset/dtable/css/jquery.css b/superset/dtable/css/jquery.css new file mode 100644 index 0000000000..9b33240136 --- /dev/null +++ b/superset/dtable/css/jquery.css @@ -0,0 +1,144 @@ +.dataTables_wrapper .dataTables_paginate { + float:right; + text-align:right; + padding-top:0.25em +} +.dataTables_wrapper .dataTables_paginate .paginate_button { + box-sizing:border-box; + display:inline-block; + min-width:1.5em; + padding:0.5em 1em; + margin-left:2px; + text-align:center; + text-decoration:none !important; + cursor:pointer; + *cursor:hand; + color:#333 !important; + border:1px solid transparent; + border-radius:2px +} +.dataTables_wrapper .dataTables_paginate .paginate_button.current, +.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover { + color:#333 !important; + border:1px solid #979797; + background-color:white; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #fff), color-stop(100%, #dcdcdc)); + background:-webkit-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-moz-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-ms-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-o-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:linear-gradient(to bottom, #fff 0%, #dcdcdc 100%) +} +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled, +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active { + cursor:default; + color:#666 !important; + border:1px solid transparent; + background:transparent; + box-shadow:none +} +.dataTables_wrapper .dataTables_paginate .paginate_button:hover { + color:white !important; + border:1px solid #111; + background-color:#585858; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111)); + background:-webkit-linear-gradient(top, #585858 0%, #111 100%); + background:-moz-linear-gradient(top, #585858 0%, #111 100%); + background:-ms-linear-gradient(top, #585858 0%, #111 100%); + background:-o-linear-gradient(top, #585858 0%, #111 100%); + background:linear-gradient(to bottom, #585858 0%, #111 100%) +} +.dataTables_wrapper .dataTables_paginate .paginate_button:active { + outline:none; + background-color:#2b2b2b; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c)); + background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%); + box-shadow:inset 0 0 3px #111 +} +.dataTables_wrapper .dataTables_paginate .ellipsis { + padding:0 1em +} +.dataTables_wrapper .dataTables_processing { + position:absolute; + top:50%; + left:50%; + width:100%; + height:40px; + margin-left:-50%; + margin-top:-25px; + padding-top:20px; + text-align:center; + font-size:1.2em; + background-color:white; + background:-webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0))); + background:-webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%) +} +.dataTables_wrapper .dataTables_length, +.dataTables_wrapper .dataTables_filter, +.dataTables_wrapper .dataTables_info, +.dataTables_wrapper .dataTables_processing, +.dataTables_wrapper .dataTables_paginate { + color:#333 +} +.dataTables_wrapper .dataTables_scroll { + clear:both +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody { + *margin-top:-1px; + -webkit-overflow-scrolling:touch +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th, +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td { + vertical-align:middle +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th>div.dataTables_sizing, +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td>div.dataTables_sizing { + height:0; + overflow:hidden; + margin:0 !important; + padding:0 !important +} +.dataTables_wrapper.no-footer .dataTables_scrollBody { + border-bottom:1px solid #111 +} +.dataTables_wrapper.no-footer div.dataTables_scrollHead table, +.dataTables_wrapper.no-footer div.dataTables_scrollBody table { + border-bottom:none +} +.dataTables_wrapper:after { + visibility:hidden; + display:block; + content:""; + clear:both; + height:0 +} +@media screen and (max-width: 767px) { + .dataTables_wrapper .dataTables_info, + .dataTables_wrapper .dataTables_paginate { + float:none; + text-align:center + } + .dataTables_wrapper .dataTables_paginate { + margin-top:0.5em + } +} +@media screen and (max-width: 640px) { + .dataTables_wrapper .dataTables_length, + .dataTables_wrapper .dataTables_filter { + float:none; + text-align:center + } + .dataTables_wrapper .dataTables_filter { + margin-top:0.5em + } +} + diff --git a/superset/dtable/div.dataTables_sizing b/superset/dtable/div.dataTables_sizing new file mode 100644 index 0000000000..e69de29bb2 diff --git a/superset/dtable/div.dataTables_sizing, b/superset/dtable/div.dataTables_sizing, new file mode 100644 index 0000000000..e69de29bb2 diff --git a/superset/dtable/jquery.css b/superset/dtable/jquery.css new file mode 100644 index 0000000000..9b33240136 --- /dev/null +++ b/superset/dtable/jquery.css @@ -0,0 +1,144 @@ +.dataTables_wrapper .dataTables_paginate { + float:right; + text-align:right; + padding-top:0.25em +} +.dataTables_wrapper .dataTables_paginate .paginate_button { + box-sizing:border-box; + display:inline-block; + min-width:1.5em; + padding:0.5em 1em; + margin-left:2px; + text-align:center; + text-decoration:none !important; + cursor:pointer; + *cursor:hand; + color:#333 !important; + border:1px solid transparent; + border-radius:2px +} +.dataTables_wrapper .dataTables_paginate .paginate_button.current, +.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover { + color:#333 !important; + border:1px solid #979797; + background-color:white; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #fff), color-stop(100%, #dcdcdc)); + background:-webkit-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-moz-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-ms-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:-o-linear-gradient(top, #fff 0%, #dcdcdc 100%); + background:linear-gradient(to bottom, #fff 0%, #dcdcdc 100%) +} +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled, +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, +.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active { + cursor:default; + color:#666 !important; + border:1px solid transparent; + background:transparent; + box-shadow:none +} +.dataTables_wrapper .dataTables_paginate .paginate_button:hover { + color:white !important; + border:1px solid #111; + background-color:#585858; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111)); + background:-webkit-linear-gradient(top, #585858 0%, #111 100%); + background:-moz-linear-gradient(top, #585858 0%, #111 100%); + background:-ms-linear-gradient(top, #585858 0%, #111 100%); + background:-o-linear-gradient(top, #585858 0%, #111 100%); + background:linear-gradient(to bottom, #585858 0%, #111 100%) +} +.dataTables_wrapper .dataTables_paginate .paginate_button:active { + outline:none; + background-color:#2b2b2b; + background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c)); + background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%); + background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%); + box-shadow:inset 0 0 3px #111 +} +.dataTables_wrapper .dataTables_paginate .ellipsis { + padding:0 1em +} +.dataTables_wrapper .dataTables_processing { + position:absolute; + top:50%; + left:50%; + width:100%; + height:40px; + margin-left:-50%; + margin-top:-25px; + padding-top:20px; + text-align:center; + font-size:1.2em; + background-color:white; + background:-webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0))); + background:-webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:-o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%); + background:linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%) +} +.dataTables_wrapper .dataTables_length, +.dataTables_wrapper .dataTables_filter, +.dataTables_wrapper .dataTables_info, +.dataTables_wrapper .dataTables_processing, +.dataTables_wrapper .dataTables_paginate { + color:#333 +} +.dataTables_wrapper .dataTables_scroll { + clear:both +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody { + *margin-top:-1px; + -webkit-overflow-scrolling:touch +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th, +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td { + vertical-align:middle +} +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th>div.dataTables_sizing, +.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td>div.dataTables_sizing { + height:0; + overflow:hidden; + margin:0 !important; + padding:0 !important +} +.dataTables_wrapper.no-footer .dataTables_scrollBody { + border-bottom:1px solid #111 +} +.dataTables_wrapper.no-footer div.dataTables_scrollHead table, +.dataTables_wrapper.no-footer div.dataTables_scrollBody table { + border-bottom:none +} +.dataTables_wrapper:after { + visibility:hidden; + display:block; + content:""; + clear:both; + height:0 +} +@media screen and (max-width: 767px) { + .dataTables_wrapper .dataTables_info, + .dataTables_wrapper .dataTables_paginate { + float:none; + text-align:center + } + .dataTables_wrapper .dataTables_paginate { + margin-top:0.5em + } +} +@media screen and (max-width: 640px) { + .dataTables_wrapper .dataTables_length, + .dataTables_wrapper .dataTables_filter { + float:none; + text-align:center + } + .dataTables_wrapper .dataTables_filter { + margin-top:0.5em + } +} + diff --git a/superset/dtable/mod_tables/__init__.py b/superset/dtable/mod_tables/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/superset/dtable/mod_tables/controllers.py b/superset/dtable/mod_tables/controllers.py new file mode 100644 index 0000000000..9ad8ab8af6 --- /dev/null +++ b/superset/dtable/mod_tables/controllers.py @@ -0,0 +1,59 @@ +from flask import Blueprint, jsonify, request, redirect, url_for +from superset.dtable import table_builder +from flask_appbuilder.security.decorators import has_access +import requests +import json +import re +import urllib.request +from bs4 import BeautifulSoup +from superset.extensions import appbuilder +from flask_oidc import OpenIDConnect +from flask_login import current_user + +tables = Blueprint('tables', __name__, url_prefix='/tables') + + +@tables.route("/serverside_table", methods=['GET']) +def serverside_table_content(): + sm = appbuilder.sm + oidc = sm.oid + +# @appbuilder.sm.oid.require_login + def handle_login(): +# if oidc.user_loggedin: + if current_user.is_authenticated: + data = table_builder.collect_data_serverside(request) + return jsonify(data) + else: + return redirect('https://superset.nets-analytics.net/login') + return handle_login() + + +@tables.route("/get_columns", methods=['GET']) +def get_sql_columns(): + sm = appbuilder.sm + oidc = sm.oid + +# @appbuilder.sm.oid.require_login + def handle_login_2(): +# if oidc.user_loggedin: + if current_user.is_authenticated: + print("User logged in") + s_query = request.args.get('sql_query') + print(s_query) + chart_id = request.args.get('chart_id') + column_list = "[ " + column_list_return= "" + for line in s_query.splitlines(): + if line == "FROM": + break + column = line.partition("\" AS ") [2] + column = column[:-1] + column_list += "{ \"data\" : " + column + "} ," + column_list_return = column_list[:-3] + "\" } ]" + else: + print("No user logged in") + column_list_return = redirect('https://superset.nets-analytics.net/login') + return column_list_return + + return handle_login_2() diff --git a/superset/dtable/mod_tables/models.py b/superset/dtable/mod_tables/models.py new file mode 100644 index 0000000000..237b36b04b --- /dev/null +++ b/superset/dtable/mod_tables/models.py @@ -0,0 +1,96 @@ +from flask import Blueprint, jsonify, request +from superset.dtable.mod_tables.serverside.serverside_table import ServerSideTable +from superset.dtable.mod_tables.serverside import table_schemas +import requests +import json +import re +import urllib.request +from bs4 import BeautifulSoup + +class TableBuilder(object): + + def druid_data(self,request): + + print("Generate SQL") + chart_id = self.request_values["chart_id"] + s_query = self.request_values["sql_query"] + print(s_query) + curr_line = -1 + length = s_query.count('\n') + druid_sql = "" + where= " WHERE 1=1" + for line in s_query.splitlines(): + curr_line += 1 + if line.startswith("WHERE "): + where = " " + if curr_line == length: + if line.startswith("LIMIT "): + print("LIMIT removed") + elif curr_line == 0: + druid_sql += line + else: + druid_sql += line + druid_sql = druid_sql.replace('"','\\"') + druid_sql += where + self.request_values = request.values + print("Druid SQL Column List Get") + columns = self.create_column_config(request) + + print('select done') + for i in range(len(columns)): + search_col = 'sSearch_' + str(i) + druid_col = columns[i]["data_name"] + if self.request_values[search_col]: + druid_sql += 'AND \\"' + columns[i]["data_name"] + '\\"' + " = \'" + self.request_values[search_col] + "\' " + + if self.request_values["query_limit"].isdigit(): + query_limit = self.request_values["query_limit"] + else: + query_limit = 10000 + + print(self.request_values["query_limit"].isdigit()) + druid_sql += " LIMIT " + str(query_limit) + druid_sql = "{ \"query\" : " + "\"" + druid_sql + "\" }" + print(druid_sql) + druid_obj = json.loads(druid_sql) + druid_request = "https://druid-query.internal.nets-analytics.net:8282/druid/v2/sql/" + rp=requests.post(druid_request, json = druid_obj) + return rp.json() + + def create_column_config(self, request): + print("Create column list") + self.request_values = request.values + chart_id = self.request_values["chart_id"] + s_query = self.request_values["sql_query"] + column_list = "[ " + order = 1 + last_line = 0 + length = len(s_query.splitlines()) + + for line in s_query.splitlines(): + if line == "FROM": + break + last_line += 1 + + for line in s_query.splitlines(): + if line == "FROM": + break + column = line.partition("\" AS ") [2] + + if order != last_line: + column = column[:-1] + + column_list += "{ \"data_name\":" + column + " , \"column_name\": " + column + " , \"default\": \"\", \"order\": " + str(order) + " , \"searchable\": true } ," + order += 1 + columns = column_list[:-3] + " } ]" + columns = json.loads(columns) + + return columns + + + def collect_data_clientside(self): + return {'data': DATA_SAMPLE} + + def collect_data_serverside(self, request): + self.request_values = request.values + return ServerSideTable(request, self.druid_data(request), self.create_column_config(request)).output_result() diff --git a/superset/dtable/mod_tables/serverside/__init__.py b/superset/dtable/mod_tables/serverside/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/superset/dtable/mod_tables/serverside/serverside_table.py b/superset/dtable/mod_tables/serverside/serverside_table.py new file mode 100644 index 0000000000..9cb1bc2142 --- /dev/null +++ b/superset/dtable/mod_tables/serverside/serverside_table.py @@ -0,0 +1,261 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +import re +import json + + +class ServerSideTable(object): + + ''' + Retrieves the values specified by Datatables in the request and processes + the data that will be displayed in the table (filtering, sorting and + selecting a subset of it). + + Attributes: + request: Values specified by DataTables in the request. + data: Data to be displayed in the table. + column_list: Schema of the table that will be built. It contains + the name of each column (both in the data and in the + table), the default values (if available) and the + order in the HTML. + ''' + + def __init__( + self, + request, + data, + column_list, + ): + self.result_data = None + self.cardinality_filtered = 0 + self.cardinality = 0 + + self.request_values = request.values + self.columns = sorted(column_list, key=lambda col: col['order']) + + rows = self._extract_rows_from_data(data) + self._run(rows) + + def _run(self, data): + ''' + Prepares the data, and values that will be generated as output. + It does the actual filtering, sorting and paging of the data. + + Args: + data: Data to be displayed by DataTables. + ''' + + self.cardinality = len(data) # Total num. of rows + + filtered_data = self._custom_filter(data) + self.cardinality_filtered = len(filtered_data) # Num. displayed rows + + sorted_data = self._custom_sort(filtered_data) + self.result_data = self._custom_paging(sorted_data) + + def _extract_rows_from_data(self, data): + ''' + Extracts the value of each column from the original data using the + schema of the table. + + Args: + data: Data to be displayed by DataTables. + + Returns: + List of dicts that represents the table's rows. + ''' + + rows = [] + for x in data: + row = {} + for column in self.columns: + default = column['default'] + data_name = column['data_name'] + column_name = column['column_name'] + row[column_name] = x.get(data_name, default) + rows.append(row) + return rows + + def _custom_filter(self, data): + ''' + Filters out those rows that do not contain the values specified by the + user using a case-insensitive regular expression. + + It takes into account only those columns that are 'searchable'. + + Args: + data: Data to be displayed by DataTables. + + Returns: + Filtered data. + ''' + + def check_row(row): + ''' Checks whether a row should be displayed or not. ''' + + for i in range(len(self.columns)): + if self.columns[i]['searchable']: + value = row[self.columns[i]['column_name']] + regex = '(?i)' + self.request_values['sSearch'] + if re.compile(regex).search(str(value)): + return True + return False + + def multi_check_row(row): + ''' Checks whether a row should be displayed or not. ''' + matched=0 + for i in range(len(self.columns)): + search_col = 'sSearch_' + str(i) + if self.columns[i]['searchable']: + if self.request_values.get(search_col, ''): + value = row[self.columns[i]['column_name']] + regex = '(?i)' + self.request_values[search_col] + if re.compile(regex).search(str(value)) and self.request_values[search_col]: + matched =1 + else : + return False + + if matched == 1 : + return True + else: + return False + + def glob_multi_check_row(row): + ''' Checks whether a row should be displayed or not. ''' + for i in range(len(self.columns)): + if self.columns[i]['searchable']: + col_name = 'sSearch_0' + value = row[self.columns[i]['column_name']] + regex = '(?i)' + self.request_values[col_name] + if re.compile(regex).search(str(value)): + return True + else: + value = row[self.columns[i]['column_name']] + regex = '(?i)' + self.request_values['sSearch'] + if re.compile(regex).search(str(value)): + return True + return False + + filter = 0 + + for i in range(len(self.columns)): + search_col = 'sSearch_' + str(i) + if self.columns[i]['searchable'] and filter < 2: + if self.request_values.get(search_col, ''): + if self.request_values.get('sSearch', ''): + filter = 3 + else: + filter = 1 + elif self.request_values.get('sSearch', ''): + filter = 1 + + + if filter == 1: + print("check_row") + return [row for row in data if check_row(row)] + elif filter == 2: + print("multi_check_row") + return [row for row in data if multi_check_row(row)] + elif filter == 3: + print("glob_multi_check_row") + return [row for row in data if glob_multi_check_row(row)] + else: + print("no filter selected") + return data + + def _custom_sort(self, data): + ''' + Sorts the rows taking in to account the column (or columns) that the + user has selected. + + Args: + data: Filtered data. + + Returns: + Sorted data by the columns specified by the user. + ''' + + def is_reverse(str_direction): + ''' Maps the 'desc' and 'asc' words to True or False. ''' + + return (True if str_direction == 'desc' else False) + + if self.request_values['iSortCol_0'] != '' \ + and int(self.request_values['iSortingCols']) > 0: + for i in range(0, int(self.request_values['iSortingCols'])): + column_number = int(self.request_values['iSortCol_' + + str(i)]) + column_name = self.columns[column_number]['column_name'] + sort_direction = self.request_values['sSortDir_' + + str(i)] + data = sorted(data, key=lambda x: x[column_name], + reverse=is_reverse(sort_direction)) + + return data + else: + return data + + def _custom_paging(self, data): + ''' + Selects a subset of the filtered and sorted data based on if the table + has pagination, the current page and the size of each page. + + Args: + data: Filtered and sorted data. + + Returns: + Subset of the filtered and sorted data that will be displayed by + the DataTables if the pagination is enabled. + ''' + + def requires_pagination(): + ''' Check if the table is going to be paginated ''' + + if self.request_values['iDisplayStart'] != '': + if int(self.request_values['iDisplayLength']) != -1: + return True + return False + + if not requires_pagination(): + return data + + start = int(self.request_values['iDisplayStart']) + length = int(self.request_values['iDisplayLength']) + + # if search returns only one page + + if len(data) <= length: + + # display only one page + + return data[start:] + else: + limit = -len(data) + start + length + if limit < 0: + + # display pagination + + return data[start:limit] + else: + + # display last page of pagination + + return data[start:] + + def output_result(self): + ''' + Generates a dict with the content of the response. It contains the + required values by DataTables (echo of the reponse and cardinality + values) and the data that will be displayed. + + Return: + Content of the response. + ''' + + output = {} + output['sEcho'] = str(int(self.request_values['sEcho'])) + output['iTotalRecords'] = str(self.cardinality) + output['iTotalDisplayRecords'] = str(self.cardinality_filtered) + output['data'] = self.result_data + return output + diff --git a/superset/dtable/static/js/get_columns.py b/superset/dtable/static/js/get_columns.py new file mode 100644 index 0000000000..9db3640317 --- /dev/null +++ b/superset/dtable/static/js/get_columns.py @@ -0,0 +1,11 @@ +import requests +import json +import re +import urllib.request + +resp=requests.get('http://localhost:8088/superset/explore_json/?form_data=%7B"slice_id"%3A19%7D') +resp_dta=resp.json() +array= resp_dta['query'] +print(str.array)) + +return array diff --git a/superset/dtable/static/js/serverside_table.js b/superset/dtable/static/js/serverside_table.js new file mode 100644 index 0000000000..6dad1912d4 --- /dev/null +++ b/superset/dtable/static/js/serverside_table.js @@ -0,0 +1,69 @@ +$(document).ready(function() { + + +const queryString = window.location.search; +const urlParams = new URLSearchParams(queryString); +const sliceId = parseInt(urlParams.get('slice_id')) + +function getSql() { + return sqlQuery +} + +//var chrt_id = parseInt( $('#chart_id').val(), 10 ) +console.log("Check inserted value") +console.log(getSql()) + +if (typeof(sliceId) !== 'undefined') { +$.getJSON('/tables/get_columns',{ chart_id : sliceId } , function(sqlcolumns) { + + sqlcolumns.forEach(function(col_n, index) { + $('#serverside_table thead tr:eq(0)').append(""+ col_n.data +"") + + $('#serverside_table thead tr:eq(1)').append(""+ '' +"") + }); +var table = $('#serverside_table').DataTable({ + bProcessing: true, + bServerSide: true, + sPaginationType: "full_numbers", + lengthMenu: [[10, 25, 50, 100], [10, 25, 50, 100]], + renderer: { "header": "bootstrap" }, + bjQueryUI: true, + orderCellsTop: true, + dom: 'lBfrtip', + colReorder: true, + stateSave: true, + buttons: [ + 'csv', 'excel' + ], + sAjaxSource: '/tables/serverside_table', + fnServerParams: function ( aoData ) { var limit = parseInt( $('#limit').val(), 10 ); const queryString = window.location.search; const urlParams = new URLSearchParams(queryString); const sliceId = parseInt(urlParams.get('slice_id')) ; aoData.push( { 'name': 'query_limit', 'value':limit });aoData.push( { 'name': 'chart_id', 'value':sliceId });}, + columns:sqlcolumns, + initComplete: function () { + // Apply the search + this.api().columns().every( function () { + var that = this; + $('.dataTables_scrollBody thead tr').css({visibility:'collapse'}); + } + ); +} + +} ); + + table.columns().every(function (index) { + $('#serverside_table thead tr:eq(1) th:eq(' + index + ') input').on('keyup change', function () { + table.column($(this).parent().index() + ':visible') + .search(this.value) + .draw(); + }); + }); + +$('#limit').keyup( function() { + table.draw(); + } ); + +}); +} +else { + return +} +}); diff --git a/superset/dtable/templates/css/jquery.css b/superset/dtable/templates/css/jquery.css new file mode 100644 index 0000000000..781de6bfc4 --- /dev/null +++ b/superset/dtable/templates/css/jquery.css @@ -0,0 +1 @@ +table.dataTable{width:100%;margin:0 auto;clear:both;border-collapse:separate;border-spacing:0}table.dataTable thead th,table.dataTable tfoot th{font-weight:bold}table.dataTable thead th,table.dataTable thead td{padding:10px 18px;border-bottom:1px solid #111}table.dataTable thead th:active,table.dataTable thead td:active{outline:none}table.dataTable tfoot th,table.dataTable tfoot td{padding:10px 18px 6px 18px;border-top:1px solid #111}table.dataTable thead .sorting,table.dataTable thead .sorting_asc,table.dataTable thead .sorting_desc{cursor:pointer;*cursor:hand}table.dataTable thead .sorting,table.dataTable thead .sorting_asc,table.dataTable thead .sorting_desc,table.dataTable thead .sorting_asc_disabled,table.dataTable thead .sorting_desc_disabled{background-repeat:no-repeat;background-position:center right}table.dataTable thead .sorting{background-image:url("../images/sort_both.png")}table.dataTable thead .sorting_asc{background-image:url("../images/sort_asc.png")}table.dataTable thead .sorting_desc{background-image:url("../images/sort_desc.png")}table.dataTable thead .sorting_asc_disabled{background-image:url("../images/sort_asc_disabled.png")}table.dataTable thead .sorting_desc_disabled{background-image:url("../images/sort_desc_disabled.png")}table.dataTable tbody tr{background-color:#ffffff}table.dataTable tbody tr.selected{background-color:#B0BED9}table.dataTable tbody th,table.dataTable tbody td{padding:8px 10px}table.dataTable.row-border tbody th,table.dataTable.row-border tbody td,table.dataTable.display tbody th,table.dataTable.display tbody td{border-top:1px solid #ddd}table.dataTable.row-border tbody tr:first-child th,table.dataTable.row-border tbody tr:first-child td,table.dataTable.display tbody tr:first-child th,table.dataTable.display tbody tr:first-child td{border-top:none}table.dataTable.cell-border tbody th,table.dataTable.cell-border tbody td{border-top:1px solid #ddd;border-right:1px solid #ddd}table.dataTable.cell-border tbody tr th:first-child,table.dataTable.cell-border tbody tr td:first-child{border-left:1px solid #ddd}table.dataTable.cell-border tbody tr:first-child th,table.dataTable.cell-border tbody tr:first-child td{border-top:none}table.dataTable.stripe tbody tr.odd,table.dataTable.display tbody tr.odd{background-color:#f9f9f9}table.dataTable.stripe tbody tr.odd.selected,table.dataTable.display tbody tr.odd.selected{background-color:#acbad4}table.dataTable.hover tbody tr:hover,table.dataTable.display tbody tr:hover{background-color:#f6f6f6}table.dataTable.hover tbody tr:hover.selected,table.dataTable.display tbody tr:hover.selected{background-color:#aab7d1}table.dataTable.order-column tbody tr>.sorting_1,table.dataTable.order-column tbody tr>.sorting_2,table.dataTable.order-column tbody tr>.sorting_3,table.dataTable.display tbody tr>.sorting_1,table.dataTable.display tbody tr>.sorting_2,table.dataTable.display tbody tr>.sorting_3{background-color:#fafafa}table.dataTable.order-column tbody tr.selected>.sorting_1,table.dataTable.order-column tbody tr.selected>.sorting_2,table.dataTable.order-column tbody tr.selected>.sorting_3,table.dataTable.display tbody tr.selected>.sorting_1,table.dataTable.display tbody tr.selected>.sorting_2,table.dataTable.display tbody tr.selected>.sorting_3{background-color:#acbad5}table.dataTable.display tbody tr.odd>.sorting_1,table.dataTable.order-column.stripe tbody tr.odd>.sorting_1{background-color:#f1f1f1}table.dataTable.display tbody tr.odd>.sorting_2,table.dataTable.order-column.stripe tbody tr.odd>.sorting_2{background-color:#f3f3f3}table.dataTable.display tbody tr.odd>.sorting_3,table.dataTable.order-column.stripe tbody tr.odd>.sorting_3{background-color:whitesmoke}table.dataTable.display tbody tr.odd.selected>.sorting_1,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_1{background-color:#a6b4cd}table.dataTable.display tbody tr.odd.selected>.sorting_2,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_2{background-color:#a8b5cf}table.dataTable.display tbody tr.odd.selected>.sorting_3,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_3{background-color:#a9b7d1}table.dataTable.display tbody tr.even>.sorting_1,table.dataTable.order-column.stripe tbody tr.even>.sorting_1{background-color:#fafafa}table.dataTable.display tbody tr.even>.sorting_2,table.dataTable.order-column.stripe tbody tr.even>.sorting_2{background-color:#fcfcfc}table.dataTable.display tbody tr.even>.sorting_3,table.dataTable.order-column.stripe tbody tr.even>.sorting_3{background-color:#fefefe}table.dataTable.display tbody tr.even.selected>.sorting_1,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_1{background-color:#acbad5}table.dataTable.display tbody tr.even.selected>.sorting_2,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_2{background-color:#aebcd6}table.dataTable.display tbody tr.even.selected>.sorting_3,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_3{background-color:#afbdd8}table.dataTable.display tbody tr:hover>.sorting_1,table.dataTable.order-column.hover tbody tr:hover>.sorting_1{background-color:#eaeaea}table.dataTable.display tbody tr:hover>.sorting_2,table.dataTable.order-column.hover tbody tr:hover>.sorting_2{background-color:#ececec}table.dataTable.display tbody tr:hover>.sorting_3,table.dataTable.order-column.hover tbody tr:hover>.sorting_3{background-color:#efefef}table.dataTable.display tbody tr:hover.selected>.sorting_1,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_1{background-color:#a2aec7}table.dataTable.display tbody tr:hover.selected>.sorting_2,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_2{background-color:#a3b0c9}table.dataTable.display tbody tr:hover.selected>.sorting_3,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_3{background-color:#a5b2cb}table.dataTable.no-footer{border-bottom:1px solid #111}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}table.dataTable.compact thead th,table.dataTable.compact thead td{padding:4px 17px 4px 4px}table.dataTable.compact tfoot th,table.dataTable.compact tfoot td{padding:4px}table.dataTable.compact tbody th,table.dataTable.compact tbody td{padding:4px}table.dataTable th.dt-left,table.dataTable td.dt-left{text-align:left}table.dataTable th.dt-center,table.dataTable td.dt-center,table.dataTable td.dataTables_empty{text-align:center}table.dataTable th.dt-right,table.dataTable td.dt-right{text-align:right}table.dataTable th.dt-justify,table.dataTable td.dt-justify{text-align:justify}table.dataTable th.dt-nowrap,table.dataTable td.dt-nowrap{white-space:nowrap}table.dataTable thead th.dt-head-left,table.dataTable thead td.dt-head-left,table.dataTable tfoot th.dt-head-left,table.dataTable tfoot td.dt-head-left{text-align:left}table.dataTable thead th.dt-head-center,table.dataTable thead td.dt-head-center,table.dataTable tfoot th.dt-head-center,table.dataTable tfoot td.dt-head-center{text-align:center}table.dataTable thead th.dt-head-right,table.dataTable thead td.dt-head-right,table.dataTable tfoot th.dt-head-right,table.dataTable tfoot td.dt-head-right{text-align:right}table.dataTable thead th.dt-head-justify,table.dataTable thead td.dt-head-justify,table.dataTable tfoot th.dt-head-justify,table.dataTable tfoot td.dt-head-justify{text-align:justify}table.dataTable thead th.dt-head-nowrap,table.dataTable thead td.dt-head-nowrap,table.dataTable tfoot th.dt-head-nowrap,table.dataTable tfoot td.dt-head-nowrap{white-space:nowrap}table.dataTable tbody th.dt-body-left,table.dataTable tbody td.dt-body-left{text-align:left}table.dataTable tbody th.dt-body-center,table.dataTable tbody td.dt-body-center{text-align:center}table.dataTable tbody th.dt-body-right,table.dataTable tbody td.dt-body-right{text-align:right}table.dataTable tbody th.dt-body-justify,table.dataTable tbody td.dt-body-justify{text-align:justify}table.dataTable tbody th.dt-body-nowrap,table.dataTable tbody td.dt-body-nowrap{white-space:nowrap}table.dataTable,table.dataTable th,table.dataTable td{-webkit-box-sizing:content-box;box-sizing:content-box}.dataTables_wrapper{position:relative;clear:both;*zoom:1;zoom:1}.dataTables_wrapper .dataTables_length{float:left}.dataTables_wrapper .dataTables_filter{float:right;text-align:right}.dataTables_wrapper .dataTables_filter input{margin-left:0.5em}.dataTables_wrapper .dataTables_info{clear:both;float:left;padding-top:0.755em}.dataTables_wrapper .dataTables_paginate{float:right;text-align:right;padding-top:0.25em}.dataTables_wrapper .dataTables_paginate .paginate_button{box-sizing:border-box;display:inline-block;min-width:1.5em;padding:0.5em 1em;margin-left:2px;text-align:center;text-decoration:none !important;cursor:pointer;*cursor:hand;color:#333 !important;border:1px solid transparent;border-radius:2px}.dataTables_wrapper .dataTables_paginate .paginate_button.current,.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{color:#333 !important;border:1px solid #979797;background-color:white;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #fff), color-stop(100%, #dcdcdc));background:-webkit-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-moz-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-ms-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-o-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:linear-gradient(to bottom, #fff 0%, #dcdcdc 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button.disabled,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{cursor:default;color:#666 !important;border:1px solid transparent;background:transparent;box-shadow:none}.dataTables_wrapper .dataTables_paginate .paginate_button:hover{color:white !important;border:1px solid #111;background-color:#585858;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));background:-webkit-linear-gradient(top, #585858 0%, #111 100%);background:-moz-linear-gradient(top, #585858 0%, #111 100%);background:-ms-linear-gradient(top, #585858 0%, #111 100%);background:-o-linear-gradient(top, #585858 0%, #111 100%);background:linear-gradient(to bottom, #585858 0%, #111 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button:active{outline:none;background-color:#2b2b2b;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);box-shadow:inset 0 0 3px #111}.dataTables_wrapper .dataTables_paginate .ellipsis{padding:0 1em}.dataTables_wrapper .dataTables_processing{position:absolute;top:50%;left:50%;width:100%;height:40px;margin-left:-50%;margin-top:-25px;padding-top:20px;text-align:center;font-size:1.2em;background-color:white;background:-webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0)));background:-webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%)}.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter,.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_processing,.dataTables_wrapper .dataTables_paginate{color:#333}.dataTables_wrapper .dataTables_scroll{clear:both}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody{*margin-top:-1px;-webkit-overflow-scrolling:touch}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td{vertical-align:middle}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td>div.dataTables_sizing{height:0;overflow:hidden;margin:0 !important;padding:0 !important}.dataTables_wrapper.no-footer .dataTables_scrollBody{border-bottom:1px solid #111}.dataTables_wrapper.no-footer div.dataTables_scrollHead table,.dataTables_wrapper.no-footer div.dataTables_scrollBody table{border-bottom:none}.dataTables_wrapper:after{visibility:hidden;display:block;content:"";clear:both;height:0}@media screen and (max-width: 767px){.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_paginate{float:none;text-align:center}.dataTables_wrapper .dataTables_paginate{margin-top:0.5em}}@media screen and (max-width: 640px){.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter{float:none;text-align:center}.dataTables_wrapper .dataTables_filter{margin-top:0.5em}} diff --git a/superset/dtable/templates/serverside_table.html b/superset/dtable/templates/serverside_table.html new file mode 100644 index 0000000000..fcfcbc3a46 --- /dev/null +++ b/superset/dtable/templates/serverside_table.html @@ -0,0 +1,31 @@ +{% extends "template.html" %} +{% block title %} + +Serverside Table + +{% endblock %} +{% block body %} + + + +
+ +
+ + + Limit: + + +
+ + + + + + + +
+
+ + +{% endblock %} diff --git a/superset/dtable/templates/template.html b/superset/dtable/templates/template.html new file mode 100644 index 0000000000..7532e3912e --- /dev/null +++ b/superset/dtable/templates/template.html @@ -0,0 +1,27 @@ + + + + + {% block title %}{% endblock %} + + + + + + + + + + + + + + +
+ + {% block body %}{% endblock %} + +
+ + + diff --git a/superset/models/schedules.py b/superset/models/schedules.py index 57ff52d1bb..712a0ade15 100644 --- a/superset/models/schedules.py +++ b/superset/models/schedules.py @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. """Models for scheduled execution of jobs""" + import enum -from typing import Optional, Type from flask_appbuilder import Model from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String, Text from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import relationship, RelationshipProperty +from sqlalchemy.orm import relationship from superset import security_manager from superset.models.helpers import AuditMixinNullable, ImportMixin @@ -29,17 +29,17 @@ metadata = Model.metadata # pylint: disable=no-member -class ScheduleType(str, enum.Enum): +class ScheduleType(enum.Enum): slice = "slice" dashboard = "dashboard" + s3 = "s3" - -class EmailDeliveryType(str, enum.Enum): +class EmailDeliveryType(enum.Enum): attachment = "Attachment" inline = "Inline" -class SliceEmailReportFormat(str, enum.Enum): +class SliceEmailReportFormat(enum.Enum): visualization = "Visualization" data = "Raw data" @@ -50,16 +50,16 @@ class EmailSchedule: __tablename__ = "email_schedules" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # pylint: disable=invalid-name active = Column(Boolean, default=True, index=True) crontab = Column(String(50)) @declared_attr - def user_id(self) -> int: + def user_id(self): return Column(Integer, ForeignKey("ab_user.id")) @declared_attr - def user(self) -> RelationshipProperty: + def user(self): return relationship( security_manager.user_model, backref=self.__tablename__, @@ -67,10 +67,10 @@ def user(self) -> RelationshipProperty: ) recipients = Column(Text) - slack_channel = Column(Text) deliver_as_group = Column(Boolean, default=False) delivery_type = Column(Enum(EmailDeliveryType)) - + email_subject = Column(String(128)) + email_body = Column(Text) class DashboardEmailSchedule(Model, AuditMixinNullable, ImportMixin, EmailSchedule): __tablename__ = "dashboard_email_schedules" @@ -87,9 +87,33 @@ class SliceEmailSchedule(Model, AuditMixinNullable, ImportMixin, EmailSchedule): email_format = Column(Enum(SliceEmailReportFormat)) -def get_scheduler_model(report_type: ScheduleType) -> Optional[Type[EmailSchedule]]: - if report_type == ScheduleType.dashboard: +def get_scheduler_model(report_type): + if report_type == ScheduleType.dashboard.value: return DashboardEmailSchedule - if report_type == ScheduleType.slice: + elif report_type == ScheduleType.slice.value: return SliceEmailSchedule + elif report_type == ScheduleType.s3.value: + return S3ExportSchedule return None + +class S3ExportSchedule(Model, AuditMixinNullable, ImportMixin): + __tablename__ = "s3_export_schedules" + id = Column(Integer, primary_key=True) # pylint: disable=invalid-name + active = Column(Boolean, default=True, index=True) + crontab = Column(String(50)) + + @declared_attr + def user_id(self): + return Column(Integer, ForeignKey("ab_user.id")) + + @declared_attr + def user(self): + return relationship( + security_manager.user_model, + backref=self.__tablename__, + foreign_keys=[self.user_id], + ) + + slice_id = Column(Integer, ForeignKey("slices.id")) + slice = relationship("Slice", backref="s3_schedules", foreign_keys=[slice_id]) + s3_path = Column(String(50)) diff --git a/superset/models/slice.py b/superset/models/slice.py index 4f73e43afb..8428938401 100644 --- a/superset/models/slice.py +++ b/superset/models/slice.py @@ -1,3 +1,4 @@ + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -263,6 +264,18 @@ def slice_link(self) -> Markup: name = escape(self.chart) return Markup(f'{name}') + @property + def dtable_url(self): + form_data = {"slice_id": self.id} + params = parse.quote(json.dumps(form_data)) + base_url = '/datatableview/datatable/{}'.format(self.id) + "?slice_id=" + str(self.id) + "&datasource_id=" + str(self.datasource_id) + "&datasource_type=" + str(self.datasource_type) + return f"{base_url}&form_data={params}" + + @property + def dtable_link(self): + url = self.dtable_url + return Markup(f'DataTable') + @property def changed_by_url(self) -> str: return f"/superset/profile/{self.created_by.username}" # type: ignore diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index 9948e94639..678e5de636 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -16,19 +16,19 @@ # under the License. """Utility functions used across Superset""" - +import re +import boto3 +import time import logging import time import urllib.request from collections import namedtuple from datetime import datetime, timedelta from email.utils import make_msgid, parseaddr -from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from urllib.error import URLError # pylint: disable=ungrouped-imports import croniter import simplejson as json -from celery.app.task import Task from dateutil.tz import tzlocal from flask import render_template, Response, session, url_for from flask_babel import gettext as __ @@ -41,104 +41,67 @@ # Superset framework imports from superset import app, db, security_manager from superset.extensions import celery_app -from superset.models.dashboard import Dashboard from superset.models.schedules import ( EmailDeliveryType, get_scheduler_model, ScheduleType, SliceEmailReportFormat, + S3ExportSchedule, ) -from superset.models.slice import Slice -from superset.tasks.slack_util import deliver_slack_msg from superset.utils.core import get_email_address_list, send_email_smtp -if TYPE_CHECKING: - # pylint: disable=unused-import - from werkzeug.datastructures import TypeConversionDict - - # Globals config = app.config logger = logging.getLogger("tasks.email_reports") logger.setLevel(logging.INFO) -EMAIL_PAGE_RENDER_WAIT = config["EMAIL_PAGE_RENDER_WAIT"] -WEBDRIVER_BASEURL = config["WEBDRIVER_BASEURL"] -WEBDRIVER_BASEURL_USER_FRIENDLY = config["WEBDRIVER_BASEURL_USER_FRIENDLY"] - -ReportContent = namedtuple( - "EmailContent", - [ - "body", # email body - "data", # attachments - "images", # embedded images for the email - "slack_message", # html not supported, only markdown - # attachments for the slack message, embedding not supported - "slack_attachment", - ], -) +# Time in seconds, we will wait for the page to load and render +PAGE_RENDER_WAIT = 30 -def _get_email_to_and_bcc( - recipients: str, deliver_as_group: bool -) -> Iterator[Tuple[str, str]]: +EmailContent = namedtuple("EmailContent", ["body", "data", "images"]) + + +def _get_recipients(schedule): bcc = config["EMAIL_REPORT_BCC_ADDRESS"] - if deliver_as_group: - to = recipients + if schedule.deliver_as_group: + to = schedule.recipients yield (to, bcc) else: - for to in get_email_address_list(recipients): + for to in get_email_address_list(schedule.recipients): yield (to, bcc) -# TODO(bkyryliuk): move email functionality into a separate module. -def _deliver_email( # pylint: disable=too-many-arguments - recipients: str, - deliver_as_group: bool, - subject: str, - body: str, - data: Optional[Dict[str, Any]], - images: Optional[Dict[str, str]], -) -> None: - for (to, bcc) in _get_email_to_and_bcc(recipients, deliver_as_group): +def _deliver_email(schedule, subject, email): + for (to, bcc) in _get_recipients(schedule): send_email_smtp( to, - subject, - body, + schedule.email_subject, + schedule.email_body, config, - data=data, - images=images, + data=email.data, + images=email.images, bcc=bcc, mime_subtype="related", dryrun=config["SCHEDULED_EMAIL_DEBUG_MODE"], ) +def _export_s3(schedule, email): + print("///////////Exporting to s3///////////////") + s3 = boto3.resource('s3') + timestr = time.strftime("%Y%m%d%H%M%S") + file_path = "s3_export/" + timestr + "_" + schedule.slice.slice_name.replace(" ", "_") + ".csv" + object = s3.Object(schedule.s3_path, file_path) + object.put(Body=email) -def _generate_report_content( - delivery_type: EmailDeliveryType, screenshot: bytes, name: str, url: str -) -> ReportContent: - data: Optional[Dict[str, Any]] - - # how to: https://api.slack.com/reference/surfaces/formatting - slack_message = __( - """ - *%(name)s*\n - <%(url)s|Explore in Superset> - """, - name=name, - url=url, - ) - - if delivery_type == EmailDeliveryType.attachment: +def _generate_mail_content(schedule, screenshot, name, url): + if schedule.delivery_type == EmailDeliveryType.attachment: images = None + e_body = '
' + schedule.email_body + '

' data = {"screenshot.png": screenshot} - body = __( - 'Explore in Superset

', - name=name, - url=url, - ) - elif delivery_type == EmailDeliveryType.inline: + body = __( e_body ) + elif schedule.delivery_type == EmailDeliveryType.inline: # Get the domain from the 'From' address .. # and make a message id without the < > in the ends domain = parseaddr(config["SMTP_MAIL_FROM"])[1].split("@")[1] @@ -146,20 +109,13 @@ def _generate_report_content( images = {msgid: screenshot} data = None - body = __( - """ - Explore in Superset

- - """, - name=name, - url=url, - msgid=msgid, - ) + e_body = '
'+ schedule.email_body + """

""" + body = __( e_body ) - return ReportContent(body, data, images, slack_message, screenshot) + return EmailContent(body, data, images) -def _get_auth_cookies() -> List["TypeConversionDict[Any, Any]"]: +def _get_auth_cookies(): # Login with the user specified to get the reports with app.test_request_context(): user = security_manager.find_user(config["EMAIL_REPORTS_USER"]) @@ -180,17 +136,14 @@ def _get_auth_cookies() -> List["TypeConversionDict[Any, Any]"]: return cookies -def _get_url_path(view: str, user_friendly: bool = False, **kwargs: Any) -> str: +def _get_url_path(view, **kwargs): with app.test_request_context(): - base_url = ( - WEBDRIVER_BASEURL_USER_FRIENDLY if user_friendly else WEBDRIVER_BASEURL + return urllib.parse.urljoin( + str(config["WEBDRIVER_BASEURL"]), url_for(view, **kwargs) ) - return urllib.parse.urljoin(str(base_url), url_for(view, **kwargs)) -def create_webdriver() -> Union[ - chrome.webdriver.WebDriver, firefox.webdriver.WebDriver -]: +def create_webdriver(): # Create a webdriver for use in fetching reports if config["EMAIL_REPORTS_WEBDRIVER"] == "firefox": driver_class = firefox.webdriver.WebDriver @@ -228,9 +181,7 @@ def create_webdriver() -> Union[ return driver -def destroy_webdriver( - driver: Union[chrome.webdriver.WebDriver, firefox.webdriver.WebDriver] -) -> None: +def destroy_webdriver(driver): """ Destroy a driver """ @@ -247,38 +198,26 @@ def destroy_webdriver( pass -def deliver_dashboard( - dashboard_id: int, - recipients: Optional[str], - slack_channel: Optional[str], - delivery_type: EmailDeliveryType, - deliver_as_group: bool, -) -> None: - +def deliver_dashboard(schedule): """ Given a schedule, delivery the dashboard as an email report """ - dashboard = db.session.query(Dashboard).filter_by(id=dashboard_id).one() + dashboard = schedule.dashboard - dashboard_url = _get_url_path( - "Superset.dashboard", dashboard_id_or_slug=dashboard.id - ) - dashboard_url_user_friendly = _get_url_path( - "Superset.dashboard", user_friendly=True, dashboard_id_or_slug=dashboard.id - ) + dashboard_url = _get_url_path("Superset.dashboard", dashboard_id=dashboard.id) # Create a driver, fetch the page, wait for the page to render driver = create_webdriver() window = config["WEBDRIVER_WINDOW"]["dashboard"] driver.set_window_size(*window) driver.get(dashboard_url) - time.sleep(EMAIL_PAGE_RENDER_WAIT) + time.sleep(PAGE_RENDER_WAIT) # Set up a function to retry once for the element. # This is buggy in certain selenium versions with firefox driver get_element = getattr(driver, "find_element_by_class_name") element = retry_call( - get_element, fargs=["grid-container"], tries=2, delay=EMAIL_PAGE_RENDER_WAIT + get_element, fargs=["grid-container"], tries=2, delay=PAGE_RENDER_WAIT ) try: @@ -291,46 +230,24 @@ def deliver_dashboard( destroy_webdriver(driver) # Generate the email body and attachments - report_content = _generate_report_content( - delivery_type, - screenshot, - dashboard.dashboard_title, - dashboard_url_user_friendly, + email = _generate_mail_content( + schedule, screenshot, dashboard.dashboard_title, dashboard_url ) - subject = __( - "%(prefix)s %(title)s", - prefix=config["EMAIL_REPORTS_SUBJECT_PREFIX"], - title=dashboard.dashboard_title, - ) + subject = __( schedule.email_body ) - if recipients: - _deliver_email( - recipients, - deliver_as_group, - subject, - report_content.body, - report_content.data, - report_content.images, - ) - if slack_channel: - deliver_slack_msg( - slack_channel, - subject, - report_content.slack_message, - report_content.slack_attachment, - ) + _deliver_email(schedule, subject, email) -def _get_slice_data(slc: Slice, delivery_type: EmailDeliveryType) -> ReportContent: +def _get_slice_data(schedule, report_type=None): + slc = schedule.slice + slice_url = _get_url_path( "Superset.explore_json", csv="true", form_data=json.dumps({"slice_id": slc.id}) ) # URL to include in the email - slice_url_user_friendly = _get_url_path( - "Superset.slice", slice_id=slc.id, user_friendly=True - ) + url = _get_url_path("Superset.slice", slice_id=slc.id) cookies = {} for cookie in _get_auth_cookies(): @@ -344,58 +261,48 @@ def _get_slice_data(slc: Slice, delivery_type: EmailDeliveryType) -> ReportConte # TODO: Move to the csv module content = response.read() + print("//////////////Creating CSV//////////////////") + if report_type == ScheduleType.s3.value: + # data = {__("%(name)s.csv", name=slc.slice_name): content} + data = content + e_body = '
 N/A

' + body = __( e_body ) + return data rows = [r.split(b",") for r in content.splitlines()] - - if delivery_type == EmailDeliveryType.inline: + if schedule.delivery_type == EmailDeliveryType.inline: data = None # Parse the csv file and generate HTML columns = rows.pop(0) - with app.app_context(): # type: ignore + with app.app_context(): body = render_template( "superset/reports/slice_data.html", columns=columns, rows=rows, name=slc.slice_name, - link=slice_url_user_friendly, + link=url, ) - elif delivery_type == EmailDeliveryType.attachment: + elif schedule.delivery_type == EmailDeliveryType.attachment: data = {__("%(name)s.csv", name=slc.slice_name): content} - body = __( - 'Explore in Superset

', - name=slc.slice_name, - url=slice_url_user_friendly, - ) + e_body = '
' + str(schedule.email_body) + '

' + body = __( e_body ) - # how to: https://api.slack.com/reference/surfaces/formatting - slack_message = __( - """ - *%(slice_name)s*\n - <%(slice_url_user_friendly)s|Explore in Superset> - """, - slice_name=slc.slice_name, - slice_url_user_friendly=slice_url_user_friendly, - ) + return EmailContent(body, data, None) - return ReportContent(body, data, None, slack_message, content) +def _get_slice_visualization(schedule): + slc = schedule.slice -def _get_slice_visualization( - slc: Slice, delivery_type: EmailDeliveryType -) -> ReportContent: # Create a driver, fetch the page, wait for the page to render driver = create_webdriver() window = config["WEBDRIVER_WINDOW"]["slice"] driver.set_window_size(*window) slice_url = _get_url_path("Superset.slice", slice_id=slc.id) - slice_url_user_friendly = _get_url_path( - "Superset.slice", slice_id=slc.id, user_friendly=True - ) driver.get(slice_url) - time.sleep(EMAIL_PAGE_RENDER_WAIT) + time.sleep(PAGE_RENDER_WAIT) # Set up a function to retry once for the element. # This is buggy in certain selenium versions with firefox driver @@ -403,7 +310,7 @@ def _get_slice_visualization( driver.find_element_by_class_name, fargs=["chart-container"], tries=2, - delay=EMAIL_PAGE_RENDER_WAIT, + delay=PAGE_RENDER_WAIT, ) try: @@ -416,53 +323,29 @@ def _get_slice_visualization( destroy_webdriver(driver) # Generate the email body and attachments - return _generate_report_content( - delivery_type, screenshot, slc.slice_name, slice_url_user_friendly - ) + return _generate_mail_content(schedule, screenshot, slc.slice_name, slice_url) -def deliver_slice( # pylint: disable=too-many-arguments - slice_id: int, - recipients: Optional[str], - slack_channel: Optional[str], - delivery_type: EmailDeliveryType, - email_format: SliceEmailReportFormat, - deliver_as_group: bool, -) -> None: +def deliver_slice(schedule, report_type=None): """ Given a schedule, delivery the slice as an email report """ - slc = db.session.query(Slice).filter_by(id=slice_id).one() - - if email_format == SliceEmailReportFormat.data: - report_content = _get_slice_data(slc, delivery_type) - elif email_format == SliceEmailReportFormat.visualization: - report_content = _get_slice_visualization(slc, delivery_type) + if report_type == ScheduleType.s3.value: + email = _get_slice_data(schedule, report_type) + elif schedule.email_format == SliceEmailReportFormat.data: + email = _get_slice_data(schedule) + elif schedule.email_format == SliceEmailReportFormat.visualization: + email = _get_slice_visualization(schedule) else: raise RuntimeError("Unknown email report format") - subject = __( - "%(prefix)s %(title)s", - prefix=config["EMAIL_REPORTS_SUBJECT_PREFIX"], - title=slc.slice_name, - ) - if recipients: - _deliver_email( - recipients, - deliver_as_group, - subject, - report_content.body, - report_content.data, - report_content.images, - ) - if slack_channel: - deliver_slack_msg( - slack_channel, - subject, - report_content.slack_message, - report_content.slack_attachment, - ) + if report_type == ScheduleType.s3.value: + print("Executing _export_s3") + _export_s3(schedule, email ) + else: + subject = __(schedule.email_body) + _deliver_email(schedule, subject, email) @celery_app.task( @@ -470,51 +353,33 @@ def deliver_slice( # pylint: disable=too-many-arguments bind=True, soft_time_limit=config["EMAIL_ASYNC_TIME_LIMIT_SEC"], ) -def schedule_email_report( # pylint: disable=unused-argument - task: Task, - report_type: ScheduleType, - schedule_id: int, - recipients: Optional[str] = None, - slack_channel: Optional[str] = None, -) -> None: +def schedule_email_report( + task, report_type, schedule_id, recipients=None +): # pylint: disable=unused-argument model_cls = get_scheduler_model(report_type) schedule = db.create_scoped_session().query(model_cls).get(schedule_id) - + print("Check 3") # The user may have disabled the schedule. If so, ignore this if not schedule or not schedule.active: logger.info("Ignoring deactivated schedule") return - recipients = recipients or schedule.recipients - slack_channel = slack_channel or schedule.slack_channel - logger.info( - "Starting report for slack: %s and recipients: %s.", slack_channel, recipients - ) - - if report_type == ScheduleType.dashboard: - deliver_dashboard( - schedule.dashboard_id, - recipients, - slack_channel, - schedule.delivery_type, - schedule.deliver_as_group, - ) - elif report_type == ScheduleType.slice: - deliver_slice( - schedule.slice_id, - recipients, - slack_channel, - schedule.delivery_type, - schedule.email_format, - schedule.deliver_as_group, - ) + # TODO: Detach the schedule object from the db session + if recipients is not None: + schedule.id = schedule_id + schedule.recipients = recipients + + if report_type == ScheduleType.dashboard.value: + deliver_dashboard(schedule) + elif report_type == ScheduleType.slice.value: + deliver_slice(schedule) + elif report_type == ScheduleType.s3.value: + deliver_slice(schedule, report_type) else: raise RuntimeError("Unknown report type") -def next_schedules( - crontab: str, start_at: datetime, stop_at: datetime, resolution: int = 0 -) -> Iterator[datetime]: +def next_schedules(crontab, start_at, stop_at, resolution=0): crons = croniter.croniter(crontab, start_at - timedelta(seconds=1)) previous = start_at - timedelta(days=1) @@ -534,36 +399,29 @@ def next_schedules( previous = eta -def schedule_window( - report_type: ScheduleType, start_at: datetime, stop_at: datetime, resolution: int -) -> None: +def schedule_window(report_type, start_at, stop_at, resolution): """ Find all active schedules and schedule celery tasks for each of them with a specific ETA (determined by parsing the cron schedule for the schedule) """ model_cls = get_scheduler_model(report_type) - - if not model_cls: - return None - dbsession = db.create_scoped_session() schedules = dbsession.query(model_cls).filter(model_cls.active.is_(True)) - + print("Check 1") for schedule in schedules: args = (report_type, schedule.id) - + print("Check Schedule") # Schedule the job for the specified time window for eta in next_schedules( schedule.crontab, start_at, stop_at, resolution=resolution ): + print("Checl 2") schedule_email_report.apply_async(args, eta=eta) - return None - @celery_app.task(name="email_reports.schedule_hourly") -def schedule_hourly() -> None: +def schedule_hourly(): """ Celery beat job meant to be invoked hourly """ if not config["ENABLE_SCHEDULED_EMAIL_REPORTS"]: @@ -575,5 +433,6 @@ def schedule_hourly() -> None: # Get the top of the hour start_at = datetime.now(tzlocal()).replace(microsecond=0, second=0, minute=0) stop_at = start_at + timedelta(seconds=3600) - schedule_window(ScheduleType.dashboard, start_at, stop_at, resolution) - schedule_window(ScheduleType.slice, start_at, stop_at, resolution) + schedule_window(ScheduleType.dashboard.value, start_at, stop_at, resolution) + schedule_window(ScheduleType.slice.value, start_at, stop_at, resolution) + schedule_window(ScheduleType.s3.value, start_at, stop_at, resolution) diff --git a/superset/templates/superset/add_dtable.html b/superset/templates/superset/add_dtable.html new file mode 100644 index 0000000000..e6a32a8c99 --- /dev/null +++ b/superset/templates/superset/add_dtable.html @@ -0,0 +1,32 @@ +{% extends "superset/base.html" %} + + + +{% block content %} + + + + + + + + +
+ + + Limit: + + +
+
+ + + + + + + + +
+
+{% endblock %} diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot index 7c48a309b1..4a8519c4e2 100644 --- a/superset/translations/messages.pot +++ b/superset/translations/messages.pot @@ -1220,50 +1220,6 @@ msgstr "" msgid "Changing this dataset is forbidden" msgstr "" -#: superset/tasks/schedules.py:124 -#, python-format -msgid "" -"\n" -" *%(name)s*\n" -"\n" -" <%(url)s|Explore in Superset>\n" -" " -msgstr "" - -#: superset/tasks/schedules.py:136 superset/tasks/schedules.py:365 -#, python-format -msgid "Explore in Superset

" -msgstr "" - -#: superset/tasks/schedules.py:149 -#, python-format -msgid "" -"\n" -" Explore in Superset

\n" -" \n" -" " -msgstr "" - -#: superset/tasks/schedules.py:301 superset/tasks/schedules.py:444 -#, python-format -msgid "%(prefix)s %(title)s" -msgstr "" - -#: superset/tasks/schedules.py:364 -#, python-format -msgid "%(name)s.csv" -msgstr "" - -#: superset/tasks/schedules.py:372 -#, python-format -msgid "" -"\n" -" *%(slice_name)s*\n" -"\n" -" <%(slice_url_user_friendly)s|Explore in Superset>\n" -" " -msgstr "" - #: superset/templates/appbuilder/navbar_right.html:35 msgid "New" msgstr "" diff --git a/superset/views/chart/mixin.py b/superset/views/chart/mixin.py index bc6fe9b1ee..37828e2813 100644 --- a/superset/views/chart/mixin.py +++ b/superset/views/chart/mixin.py @@ -36,7 +36,7 @@ class SliceMixin: # pylint: disable=too-few-public-methods "datasource_name", "owners", ) - list_columns = ["slice_link", "viz_type", "datasource_link", "creator", "modified"] + list_columns = ["slice_link", "viz_type", "datasource_link", "creator", "modified", "dtable_link"] order_columns = [ "slice_name", "viz_type", @@ -86,6 +86,7 @@ class SliceMixin: # pylint: disable=too-few-public-methods "slice_name": _("Name"), "table": _("Table"), "viz_type": _("Visualization Type"), + "dtable_link": _(" "), } add_form_query_rel_fields = {"dashboards": [["name", DashboardFilter, None]]} diff --git a/superset/views/chart/views.py b/superset/views/chart/views.py index 478a2e5639..43d9760eed 100644 --- a/superset/views/chart/views.py +++ b/superset/views/chart/views.py @@ -16,7 +16,7 @@ # under the License. import json -from flask_appbuilder import expose, has_access +from flask_appbuilder import expose, has_access, permission_name from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import lazy_gettext as _ @@ -65,6 +65,7 @@ def add(self) -> FlaskResponse: ), ) + @expose("/list/") @has_access def list(self) -> FlaskResponse: @@ -98,5 +99,6 @@ class SliceAsync(SliceModelView): # pylint: disable=too-many-ancestors "slice_name", "slice_url", "viz_type", + "dtable_link", ] - label_columns = {"icons": " ", "slice_link": _("Chart")} + label_columns = {"icons": " ", "slice_link": _("Chart"),"dtable_link": " "} diff --git a/superset/views/core.py b/superset/views/core.py index 2f736c3554..4a62b1beac 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -29,7 +29,7 @@ from flask import abort, flash, g, Markup, redirect, render_template, request, Response from flask_appbuilder import expose from flask_appbuilder.models.sqla.interface import SQLAInterface -from flask_appbuilder.security.decorators import has_access, has_access_api +from flask_appbuilder.security.decorators import has_access, has_access_api, permission_name from flask_appbuilder.security.sqla import models as ab_models from flask_babel import gettext as __, lazy_gettext as _ from sqlalchemy import and_, or_, select @@ -105,6 +105,7 @@ json_errors_response, json_success, validate_sqlatable, + SupersetModelView, ) from superset.views.database.filters import DatabaseFilter from superset.views.utils import ( @@ -149,12 +150,79 @@ DATASOURCE_MISSING_ERR = __("The data source seems to have been deleted") USER_MISSING_ERR = __("The user seems to have been deleted") +class DataTableView(BaseSupersetView): + + @has_access + @expose('/datatable//', methods=["GET", "POST"]) + def dt(self,slice_id): + datasource_id = request.args.get("datasource_id") + datasource_type = request.args.get("datasource_type") + slice_id_arg = request.args.get("slice_id") + print(str(slice_id_arg)) + print(datasource_type) + print(str(datasource_id)) + + response_type = utils.ChartDataResultType.QUERY + + responses: List[ + Union[utils.ChartDataResultFormat, utils.ChartDataResultType] + ] = list(utils.ChartDataResultFormat) + responses.extend(list(utils.ChartDataResultType)) + for response_option in responses: + if request.args.get(response_option) == "true": + response_type = response_option + break + + form_data = get_form_data()[0] + + try: + datasource_id, datasource_type = get_datasource_info( + datasource_id, datasource_type, form_data + ) + viz_obj = get_viz( + datasource_type=cast(str, datasource_type), + datasource_id=datasource_id, + form_data=form_data, + force=request.args.get("force") == "true", + ) + query = self.generate_json(viz_obj, response_type) + print(str(query) + "This has been printed") + except SupersetException as ex: + return json_error_response(utils.error_msg_from_exception(ex)) + + + return self.render_template("superset/add_dtable.html", slice_id=slice_id, query=str(query)) + + def generate_json( + self, viz_obj: BaseViz, response_type: Optional[str] = None + ) -> FlaskResponse: + return self.get_query_string_response(viz_obj) + + def get_query_string_response(self, viz_obj: BaseViz) -> FlaskResponse: + query = None + try: + query_obj = viz_obj.query_obj() + if query_obj: + query = viz_obj.datasource.get_query_str(query_obj) + return query + except Exception as ex: # pylint: disable=broad-except + err_msg = utils.error_msg_from_exception(ex) + logger.exception(err_msg) + return json_error_response(err_msg) + class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods """The base views for Superset!""" logger = logging.getLogger(__name__) +# @has_access +# @permission_name('dtable_access') + @expose('/dtable//', methods=["GET", "POST"]) + def dt(self,slice_id): + return self.render_template("superset/add_dtable.html", slice_id=slice_id) + + @has_access_api @expose("/datasources/") def datasources(self) -> FlaskResponse: @@ -384,10 +452,12 @@ def slice(self, slice_id: int) -> FlaskResponse: # pylint: disable=no-self-use def get_query_string_response(self, viz_obj: BaseViz) -> FlaskResponse: query = None + print("Get Queryyyyyyyyyyyyyyy") try: query_obj = viz_obj.query_obj() if query_obj: query = viz_obj.datasource.get_query_str(query_obj) + print(str(query) + "Querrrrrrrryyyyy") except Exception as ex: # pylint: disable=broad-except err_msg = utils.error_msg_from_exception(ex) logger.exception(err_msg) @@ -401,6 +471,7 @@ def get_query_string_response(self, viz_obj: BaseViz) -> FlaskResponse: ) def get_raw_results(self, viz_obj: BaseViz) -> FlaskResponse: + print("get raw result") return self.json_response( {"data": viz_obj.get_df_payload()["df"].to_dict("records")} ) @@ -411,23 +482,29 @@ def get_samples(self, viz_obj: BaseViz) -> FlaskResponse: def generate_json( self, viz_obj: BaseViz, response_type: Optional[str] = None ) -> FlaskResponse: + print("json gen start") if response_type == utils.ChartDataResultFormat.CSV: + print("csv") return CsvResponse( viz_obj.get_csv(), status=200, headers=generate_download_headers("csv"), mimetype="application/csv", - ) + ) if response_type == utils.ChartDataResultType.QUERY: + print("Query" + str(response_type)) return self.get_query_string_response(viz_obj) if response_type == utils.ChartDataResultType.RESULTS: + print("Result") return self.get_raw_results(viz_obj) if response_type == utils.ChartDataResultType.SAMPLES: + print("sample") return self.get_samples(viz_obj) + print("get payload") payload = viz_obj.get_payload() return data_payload_response(*viz_obj.payload_json_and_has_error(payload)) @@ -518,19 +595,18 @@ def explore_json( break form_data = get_form_data()[0] - + print(str(form_data)) try: datasource_id, datasource_type = get_datasource_info( datasource_id, datasource_type, form_data ) - viz_obj = get_viz( datasource_type=cast(str, datasource_type), datasource_id=datasource_id, form_data=form_data, force=request.args.get("force") == "true", ) - + print(str(viz_obj) + "Teeeeeeeeest") return self.generate_json(viz_obj, response_type) except SupersetException as ex: return json_error_response(utils.error_msg_from_exception(ex)) diff --git a/superset/views/dtable.py b/superset/views/dtable.py new file mode 100644 index 0000000000..b1396e1cfd --- /dev/null +++ b/superset/views/dtable.py @@ -0,0 +1,28 @@ +from flask import flash, g +from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access +from .base import DeleteMixin, SupersetModelView +from flask import abort, flash, g, Markup, redirect, render_template, request, Response +from flask_appbuilder import BaseView, Model, ModelView +from superset.views.base import ( + api, + BaseSupersetView, + check_ownership, + common_bootstrap_payload, + create_table_permissions, + CsvResponse, + data_payload_response, + generate_download_headers, + get_error_msg, + get_user_roles, + handle_api_exception, + json_error_response, + json_errors_response, + json_success, + validate_sqlatable, +) +class DataTableView(BaseView): + + @expose('/dtable//', methods=["GET", "POST"]) + def dt(self,slice_id): + return self.render_template("superset/add_dtable.html", slice_id=slice_id) diff --git a/superset/views/schedules.py b/superset/views/schedules.py index d98c339b60..01a316fa12 100644 --- a/superset/views/schedules.py +++ b/superset/views/schedules.py @@ -14,8 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import boto3 import enum -from typing import Type, Union +from typing import Optional, Type import simplejson as json from croniter import croniter @@ -24,7 +25,7 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_appbuilder.security.decorators import has_access from flask_babel import lazy_gettext as _ -from wtforms import BooleanField, Form, StringField +from wtforms import BooleanField, StringField from superset import db, security_manager from superset.constants import RouteMethod @@ -34,10 +35,10 @@ DashboardEmailSchedule, ScheduleType, SliceEmailSchedule, + S3ExportSchedule, ) from superset.models.slice import Slice from superset.tasks.schedules import schedule_email_report -from superset.typing import FlaskResponse from superset.utils.core import get_email_address_list, json_iso_dttm_ser from superset.views.core import json_success @@ -49,14 +50,8 @@ class EmailScheduleView( ): # pylint: disable=too-many-ancestors include_route_methods = RouteMethod.CRUD_SET _extra_data = {"test_email": False, "test_email_recipients": None} - - @property - def schedule_type(self) -> str: - raise NotImplementedError() - - @property - def schedule_type_model(self) -> Type[Union[Dashboard, Slice]]: - raise NotImplementedError() + schedule_type: Optional[Type] = None + schedule_type_model: Optional[Type] = None page_size = 20 @@ -90,32 +85,19 @@ def schedule_type_model(self) -> Type[Union[Dashboard, Slice]]: description="List of recipients to send test email to. " "If empty, we send it to the original recipients", ), - "test_slack_channel": StringField( - "Test Slack Channel", - default=None, - description="A slack channel to send a test message to.", - ), } edit_form_extra_fields = add_form_extra_fields - def process_form(self, form: Form, is_created: bool) -> None: + def process_form(self, form, is_created): if form.test_email_recipients.data: test_email_recipients = form.test_email_recipients.data.strip() else: test_email_recipients = None - - test_slack_channel = ( - form.test_slack_channel.data.strip() - if form.test_slack_channel.data - else None - ) - self._extra_data["test_email"] = form.test_email.data self._extra_data["test_email_recipients"] = test_email_recipients - self._extra_data["test_slack_channel"] = test_slack_channel - def pre_add(self, item: "EmailScheduleView") -> None: + def pre_add(self, item): try: recipients = get_email_address_list(item.recipients) item.recipients = ", ".join(recipients) @@ -126,16 +108,15 @@ def pre_add(self, item: "EmailScheduleView") -> None: if not croniter.is_valid(item.crontab): raise SupersetException("Invalid crontab format") - def pre_update(self, item: "EmailScheduleView") -> None: + def pre_update(self, item): self.pre_add(item) - def post_add(self, item: "EmailScheduleView") -> None: + def post_add(self, item): # Schedule a test mail if the user requested for it. if self._extra_data["test_email"]: recipients = self._extra_data["test_email_recipients"] or item.recipients - slack_channel = self._extra_data["test_slack_channel"] or item.slack_channel args = (self.schedule_type, item.id) - kwargs = dict(recipients=recipients, slack_channel=slack_channel) + kwargs = dict(recipients=recipients) schedule_email_report.apply_async(args=args, kwargs=kwargs) # Notify the user that schedule changes will be activate only in the @@ -143,12 +124,12 @@ def post_add(self, item: "EmailScheduleView") -> None: if item.active: flash("Schedule changes will get applied in one hour", "warning") - def post_update(self, item: "EmailScheduleView") -> None: + def post_update(self, item): self.post_add(item) @has_access @expose("/fetch//", methods=["GET"]) - def fetch_schedules(self, item_id: int) -> FlaskResponse: + def fetch_schedules(self, item_id): query = db.session.query(self.datamodel.obj) query = query.join(self.schedule_type_model).filter( @@ -177,7 +158,7 @@ def fetch_schedules(self, item_id: int) -> FlaskResponse: class DashboardEmailScheduleView( EmailScheduleView ): # pylint: disable=too-many-ancestors - schedule_type = ScheduleType.dashboard + schedule_type = ScheduleType.dashboard.value schedule_type_model = Dashboard add_title = _("Schedule Email Reports for Dashboards") @@ -201,12 +182,12 @@ class DashboardEmailScheduleView( "active", "crontab", "recipients", - "slack_channel", + "email_subject", + "email_body", "deliver_as_group", "delivery_type", "test_email", "test_email_recipients", - "test_slack_channel", ] edit_columns = add_columns @@ -227,19 +208,20 @@ class DashboardEmailScheduleView( "active": _("Active"), "crontab": _("Crontab"), "recipients": _("Recipients"), - "slack_channel": _("Slack Channel"), + "email_subject": _("Email Subject"), + "email_body": _("Email Body"), "deliver_as_group": _("Deliver As Group"), "delivery_type": _("Delivery Type"), } - def pre_add(self, item: "DashboardEmailScheduleView") -> None: + def pre_add(self, item): if item.dashboard is None: raise SupersetException("Dashboard is mandatory") super(DashboardEmailScheduleView, self).pre_add(item) class SliceEmailScheduleView(EmailScheduleView): # pylint: disable=too-many-ancestors - schedule_type = ScheduleType.slice + schedule_type = ScheduleType.slice.value schedule_type_model = Slice add_title = _("Schedule Email Reports for Charts") edit_title = add_title @@ -262,13 +244,13 @@ class SliceEmailScheduleView(EmailScheduleView): # pylint: disable=too-many-anc "active", "crontab", "recipients", - "slack_channel", + "email_subject", + "email_body", "deliver_as_group", "delivery_type", "email_format", "test_email", "test_email_recipients", - "test_slack_channel", ] edit_columns = add_columns @@ -290,13 +272,72 @@ class SliceEmailScheduleView(EmailScheduleView): # pylint: disable=too-many-anc "active": _("Active"), "crontab": _("Crontab"), "recipients": _("Recipients"), - "slack_channel": _("Slack Channel"), + "email_subject":_("Email Subject"), + "email_body": _("Email Body"), "deliver_as_group": _("Deliver As Group"), "delivery_type": _("Delivery Type"), "email_format": _("Email Format"), } - def pre_add(self, item: "SliceEmailScheduleView") -> None: + def pre_add(self, item): if item.slice is None: raise SupersetException("Slice is mandatory") super(SliceEmailScheduleView, self).pre_add(item) + +class S3ScheduleView(SupersetModelView, DeleteMixin): # pylint: disable=too-many-ancestors + add_title = _("Schedule S3 Exports for Charts") + edit_title = add_title + list_title = _("Manage S3 Exports for Charts") + schedule_type = ScheduleType.s3 + datamodel = SQLAInterface(S3ExportSchedule) + order_columns = ["user", "slice", "created_on"] + list_columns = [ + "slice", + "active", + "crontab", + "user", + "s3_path", + ] + + add_columns = [ + "slice", + "active", + "crontab", + "s3_path", + ] + + edit_columns = add_columns + + search_columns = [ + "slice", + "active", + "user", + ] + + label_columns = { + "slice": _("Chart"), + "created_on": _("Created On"), + "changed_on": _("Changed On"), + "user": _("User"), + "active": _("Active"), + "crontab": _("Crontab"), + "s3_path": _("S3 Path"), + } + + def pre_add(self, item): + if item.slice is None: + raise SupersetException("Slice is mandatory") + if item.s3_path is None: + raise SupersetException("S3 path is mandatory") + if not croniter.is_valid(item.crontab): + raise SupersetException("Invalid crontab format") +# if not str(item.s3_path).startswith("s3://"): +# raise SupersetException("Path must start with s3://") + s3 = boto3.resource('s3') + bucket = s3.Bucket(item.s3_path) + if not bucket.creation_date: + raise SupersetException("Invalid Bucket") + super(S3ScheduleView, self).pre_add(item) + + def pre_update(self, item): + self.pre_add(item)