Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Datasource URL extension. #6

Merged
merged 1 commit into from
Sep 11, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
include AUTHORS.rst CHANGELOG.rst README.rst
recursive-include src/redash_stmo *.html *.js
4 changes: 3 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
],
packages=find_packages(where='src'),
package_dir={'': 'src'},
include_package_data=True,
description="Extensions to Redash by Mozilla",
author='Mozilla Foundation',
author_email='dev-webdev@lists.mozilla.org',
Expand All @@ -21,7 +22,8 @@
entry_points={
'redash.extensions': [
'dockerflow = redash_stmo.dockerflow:dockerflow',
'datasource_health = redash_stmo.health:datasource_health'
'datasource_health = redash_stmo.health:datasource_health',
'datasource_link = redash_stmo.datasource_link:datasource_link'
],
},
classifiers=[
Expand Down
68 changes: 68 additions & 0 deletions src/redash_stmo/datasource_link.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from redash.models import DataSource
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please stay consistent with the 4 character indentation.

from redash.handlers.api import api
from redash.handlers.base import BaseResource, get_object_or_404
from redash.permissions import require_access, view_only
from redash.query_runner import BaseQueryRunner, query_runners

DATASOURCE_URLS = {
"bigquery": "https://cloud.google.com/bigquery/docs/reference/legacy-sql",
"Cassandra": "http://cassandra.apache.org/doc/latest/cql/index.html",
"dynamodb_sql": "https://dql.readthedocs.io/en/latest/",
"baseelasticsearch": "https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html",
"google_spreadsheets": "http://redash.readthedocs.io/en/latest/datasources.html#google-spreadsheets",
"hive": "https://cwiki.apache.org/confluence/display/Hive/LanguageManual",
"impala": "http://www.cloudera.com/documentation/enterprise/latest/topics/impala_langref.html",
"influxdb": "https://docs.influxdata.com/influxdb/v1.0/query_language/spec/",
"jirajql": "https://confluence.atlassian.com/jirasoftwarecloud/advanced-searching-764478330.html",
"mongodb": "https://docs.mongodb.com/manual/reference/operator/query/",
"mssql": "https://msdn.microsoft.com/en-us/library/bb510741.aspx",
"mysql": "https://dev.mysql.com/doc/refman/5.7/en/",
"oracle": "http://docs.oracle.com/database/121/SQLRF/toc.htm",
"pg": "https://www.postgresql.org/docs/current/",
"redshift": "http://docs.aws.amazon.com/redshift/latest/dg/cm_chap_SQLCommandRef.html",
"presto": "https://prestodb.io/docs/current/",
"python": "http://redash.readthedocs.io/en/latest/datasources.html#python",
"insecure_script": "http://redash.readthedocs.io/en/latest/datasources.html#python",
"sqlite": "http://sqlite.org/lang.html",
"treasuredata": "https://docs.treasuredata.com/categories/hive",
"url": "http://redash.readthedocs.io/en/latest/datasources.html#url",
"vertica": (
"https://my.vertica.com/docs/8.0.x/HTML/index.htm#Authoring/"
"ConceptsGuide/Other/SQLOverview.htm%3FTocPath%3DSQL"
"%2520Reference%2520Manual%7C_____1"
)
}


class DataSourceLinkResource(BaseResource):
def get(self, data_source_id):
data_source = get_object_or_404(
DataSource.get_by_id_and_org,
data_source_id,
self.current_org,
)
require_access(data_source.groups, self.current_user, view_only)
try:
result = {
"type_name": data_source.query_runner.name(),
"doc_url": data_source.options.get("doc_url", None)
}
except Exception as e:
return {"message": unicode(e), "ok": False}
else:
return {"message": result, "ok": True}

def datasource_link(app=None):
for runner_type, runner_class in query_runners.items():
if runner_type not in DATASOURCE_URLS:
continue

runner_class.add_configuration_property("doc_url", {
"type": "string",
"title": "Documentation URL",
"default": DATASOURCE_URLS[runner_type]})

# After api.init_app() is called, api.app should be set by Flask (but it's not) so that
# further calls to add_resource() are handled immediately for the given app.
api.app = app
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this an issue in Redash or something that Flask-Restful does wrong?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an issue in Flask-Restful. There are a few things going on that led me to adding this line.

tl;dr: Flask-Restful does not play well with the Redash setup of multiple workers each having a different instance of a flask app.

  1. Inside create_app() in redash/__init__.py, we cannot move handlers.init_app(app) (which calls api.init_app(app)) after extensions.init_extensions(app) because the datasource_health extension overwrites the status_api endpoint after it’s created. If extensions initialize first then an attempt is made in Redash to add status_api and it crashes. So we keep the init order so redash-stmo can override endpoints that already exist.

  2. Unlike api = Api(app) (https://github.com/flask-restful/flask-restful/blob/master/flask_restful/__init__.py#L92), api.init_app() does not set api.app when it's called (https://github.com/flask-restful/flask-restful/blob/master/flask_restful/__init__.py#L196). This results in any further calls to api.add_resource() after api.init_app() to actually have no effect. It results in appending to api.resources (https://github.com/flask-restful/flask-restful/blob/master/flask_restful/__init__.py#L384) but no call to register_view()

  3. Even if we did manage to work around this issue in 1., this would not resolve this issue. A Flask app is created in each worker with create_app(). If an extension tries to do api.add_org_resource(), the same resource will be added to api.resources from multiple workers. Now when api.init_app(app) is called, there is an exception thrown that there are multiple view functions with the same name.

The crux of the problem is that api.resources is shared among multiple apps as a temporary cache until api.init_app(app) is called so that resources can be attached to a given app. Since we have a different app for each worker, and each worker is adding the same resource in an extension, this model doesn't quite work.

By setting api.app = app we ensure that the resource is added to the correct app and there is no duplication by caching in api.resources.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense to me!

api.add_org_resource(DataSourceLinkResource, '/api/data_sources/<data_source_id>/link')
61 changes: 61 additions & 0 deletions src/redash_stmo/datasource_link/bundle/datasource_link.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import React from 'react';
import PropTypes from 'prop-types';
import { react2angular } from 'react2angular';

class DatasourceLink extends React.Component {
static propTypes = {
clientConfig: PropTypes.object.isRequired,
datasourceId: PropTypes.number.isRequired,
}

constructor(props) {
super(props);
this.state = {
type_name: '',
doc_url: '',
};
}

loadURLData() {
fetch(`${this.props.clientConfig.basePath}api/data_sources/${this.props.datasourceId}/link`)
.then((response) => {
if (response.status === 200) {
return response.json();
}
return {};
})
.catch(error => {
console.error(`Error loading data source URL: ${error}`);
return {};
})
.then((json) => {
const { type_name, doc_url } = json.message;
this.setState({ type_name, doc_url });
});
}

componentDidMount() {
this.loadURLData();
}

componentDidUpdate(prevProps) {
if (this.props.datasourceId !== prevProps.datasourceId) {
this.loadURLData();
}
}

render() {
if (!this.state.doc_url) {
return null;
}
return (
<span>
<a href={this.state.doc_url}> {this.state.type_name} documentation</a>
</span>
);
}
}

export default function init(ngModule) {
ngModule.component('datasourceLink', react2angular(DatasourceLink, ['datasourceId'], ['clientConfig']));
}
33 changes: 33 additions & 0 deletions tests/test_datasource_link.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import mock

from tests import BaseTestCase
from flask import Flask

from redash.models import DataSource
from redash.query_runner.pg import PostgreSQL
from redash_stmo.datasource_link import datasource_link, BaseQueryRunner


class TestDatasourceLink(BaseTestCase):
EXPECTED_DOC_URL = "www.example.com"
def setUp(self):
super(TestDatasourceLink, self).setUp()
self.patched_query_runners = self._setup_mock('redash_stmo.datasource_link.query_runners')
self.patched_query_runners.return_value = {}
datasource_link(self.app)

def _setup_mock(self, function_to_patch):
patcher = mock.patch(function_to_patch)
patched_function = patcher.start()
self.addCleanup(patcher.stop)
return patched_function

def test_gets_datasource_link_and_type(self):
admin = self.factory.create_admin()
data_source = self.factory.create_data_source()
data_source.options["doc_url"] = self.EXPECTED_DOC_URL

rv = self.make_request('get', '/api/data_sources/{}/link'.format(data_source.id), user=admin)
self.assertEqual(200, rv.status_code)
self.assertEqual(rv.json['message']['type_name'], data_source.query_runner.name())
self.assertEqual(rv.json['message']["doc_url"], self.EXPECTED_DOC_URL)