adapt to the renaming of the data-integration package to `mara-pipe…

…lines`

app/data_integration/__init__.py

@@ -3,19 +3,19 @@
 import datetime
 import functools
 
-import data_integration.config
+import mara_pipelines.config
 import etl_tools.config
-from data_integration.pipelines import Pipeline
+from mara_pipelines.pipelines import Pipeline
 from mara_app.monkey_patch import patch
 
 import app.config
 
-patch(data_integration.config.data_dir)(lambda: app.config.data_dir())
-patch(data_integration.config.first_date)(lambda: app.config.first_date())
-patch(data_integration.config.default_db_alias)(lambda: 'dwh')
+patch(mara_pipelines.config.data_dir)(lambda: app.config.data_dir())
+patch(mara_pipelines.config.first_date)(lambda: app.config.first_date())
+patch(mara_pipelines.config.default_db_alias)(lambda: 'dwh')
 
 
-@patch(data_integration.config.root_pipeline)
+@patch(mara_pipelines.config.root_pipeline)
 @functools.lru_cache(maxsize=None)
 def root_pipeline():
     import app.data_integration.pipelines.github

app/data_integration/pipelines/github/__init__.py

@@ -1,11 +1,11 @@
 import pathlib
 
 import etl_tools.utils
-from data_integration.commands.files import Compression
-from data_integration.commands.sql import ExecuteSQL
-from data_integration.parallel_tasks.files import ParallelReadFile, ReadMode
-from data_integration.parallel_tasks.sql import ParallelExecuteSQL
-from data_integration.pipelines import Pipeline, Task
+from mara_pipelines.commands.files import Compression
+from mara_pipelines.commands.sql import ExecuteSQL
+from mara_pipelines.parallel_tasks.files import ParallelReadFile, ReadMode
+from mara_pipelines.parallel_tasks.sql import ParallelExecuteSQL
+from mara_pipelines.pipelines import Pipeline, Task
 from etl_tools.create_attributes_table import CreateAttributesTable
 
 pipeline = Pipeline(

app/data_integration/pipelines/pypi/__init__.py

@@ -1,11 +1,11 @@
 import pathlib
 
 import etl_tools.utils
-from data_integration.commands.files import Compression
-from data_integration.commands.sql import ExecuteSQL
-from data_integration.parallel_tasks.files import ParallelReadFile, ReadMode
-from data_integration.parallel_tasks.sql import ParallelExecuteSQL
-from data_integration.pipelines import Pipeline, Task
+from mara_pipelines.commands.files import Compression
+from mara_pipelines.commands.sql import ExecuteSQL
+from mara_pipelines.parallel_tasks.files import ParallelReadFile, ReadMode
+from mara_pipelines.parallel_tasks.sql import ParallelExecuteSQL
+from mara_pipelines.pipelines import Pipeline, Task
 from etl_tools.create_attributes_table import CreateAttributesTable
 
 pipeline = Pipeline(

app/data_integration/pipelines/python_projects/__init__.py

@@ -1,9 +1,9 @@
 import pathlib
 
 import etl_tools.utils
-from data_integration.commands.sql import ExecuteSQL
-from data_integration.parallel_tasks.sql import ParallelExecuteSQL
-from data_integration.pipelines import Pipeline, Task
+from mara_pipelines.commands.sql import ExecuteSQL
+from mara_pipelines.parallel_tasks.sql import ParallelExecuteSQL
+from mara_pipelines.pipelines import Pipeline, Task
 from etl_tools.create_attributes_table import CreateAttributesTable
 
 

app/data_integration/pipelines/utils/__init__.py

@@ -1,7 +1,7 @@
 import pathlib
 
 from etl_tools import initialize_utils, create_time_dimensions
-from data_integration import pipelines
+from mara_pipelines import pipelines
 
 pipeline = pipelines.Pipeline(
     id="utils",

app/local_setup.py.example

@@ -1,6 +1,6 @@
 import datetime
 
-import data_integration.config
+import mara_pipelines.config
 import mara_acl.config
 import mara_db.config
 import mara_db.dbs
@@ -23,7 +23,7 @@ patch(mara_acl.config.require_email_http_header)(lambda: False)
 
 # How many cores to use for running the ETL, defaults to the number of CPUs of the machine
 # On production, make sure the ETL does not slow down other services too much
-patch(data_integration.config.max_number_of_parallel_tasks)(lambda: 4)
+patch(mara_pipelines.config.max_number_of_parallel_tasks)(lambda: 4)
 
 # The first day for which to download and process data (default 2017-01-01).
 # Locally, a few days of data is enough to test a pipeline.
@@ -33,4 +33,4 @@ patch(app.config.first_date)(lambda: datetime.date.today() - datetime.timedelta(
 
 # Whether it is possible to run the ETL from the web UI
 # Disable on production
-patch(data_integration.config.allow_run_from_web_ui)(lambda: True)
+patch(mara_pipelines.config.allow_run_from_web_ui)(lambda: True)

app/ui/__init__.py

@@ -1,6 +1,6 @@
 """Set up Navigation, ACL & Logos"""
 
-import data_integration
+import mara_pipelines
 import mara_data_explorer
 import flask
 import mara_acl
@@ -39,7 +39,7 @@ def css_files(original_function, response):
 @monkey_patch.patch(mara_acl.config.resources)
 def acl_resources():
     return [acl.AclResource(name='Documentation',
-                            children=[data_integration.MARA_ACL_RESOURCES().get('Data Integration'),
+                            children=[mara_pipelines.MARA_ACL_RESOURCES().get('Pipelines'),
                                       mara_db.MARA_ACL_RESOURCES().get('DB Schema')]),
             acl.AclResource(name='Data',
                             children=mara_data_explorer.MARA_ACL_RESOURCES().values()),
@@ -60,7 +60,7 @@ def acl_resources():
 @monkey_patch.patch(mara_app.config.navigation_root)
 def navigation_root() -> navigation.NavigationEntry:
     return navigation.NavigationEntry(label='Root', children=[
-        data_integration.MARA_NAVIGATION_ENTRIES().get('Data Integration'),
+        mara_pipelines.MARA_NAVIGATION_ENTRIES().get('Pipelines'),
         mara_data_explorer.MARA_NAVIGATION_ENTRIES().get('Explore'),
         mara_db.MARA_NAVIGATION_ENTRIES().get('DB Schema'),
         navigation.NavigationEntry(

app/ui/start_page.py

@@ -5,5 +5,5 @@
 
 @blueprint.route('/')
 def start_page():
-    return flask.redirect(flask.url_for('data_integration.node_page'))
+    return flask.redirect(flask.url_for('mara_pipelines.node_page'))
 #    return response.Response(title='Welcome', html=_.div['nothing to see yet'])

requirements.txt

@@ -4,8 +4,8 @@
 -e git+https://github.com/mara/mara-db.git@4.5.0#egg=mara-db
 -e git+https://github.com/mara/mara-page.git@1.5.1#egg=mara-page
 
--e git+https://github.com/mara/data-integration.git@2.8.3#egg=data-integration
--e git+https://github.com/mara/mara-etl-tools.git@3.1.1#egg=mara-etl-tools
+-e git+https://github.com/mara/mara-pipelines.git@3.0.0#egg=mara-pipelines
+-e git+https://github.com/mara/mara-etl-tools.git@4.0.0#egg=mara-etl-tools
 -e git+https://github.com/mara/bigquery-downloader.git@3.0.0#egg=bigquery-downloader
 -e git+https://github.com/mara/mara-data-explorer.git@3.0.0#egg=mara-data-explorer