Rc v034 (#1367)

Co-authored-by: Vikram Sreekanti <vsreekanti@gmail.com>
Co-authored-by: Wei Chen <wei@aqueducthq.com>
Co-authored-by: Ubuntu <ubuntu@ip-172-31-33-60.us-east-2.compute.internal>
Co-authored-by: Andre Giron <andre@aqueducthq.com>
Co-authored-by: Andre Giron <andre@spiralai.co>

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.3.4
+Released on May 24, 2023.
+
+### Enhancements
+* Improves consistency of icon sizing on resources page.
+* Merges `flow_id` and `name` into a single argument when retrieving workflows
+    from the SDK.
+* Adds ability to parametrize save operators when using AWS S3.
+
+### Bugfixes
+* Fixes bug where race condition could occur when syncing Airflow workflows to
+    Aqueduct.
+* Fixes bug where listing Snowflake data objects would return an error.
+
 ## 0.3.3
 Released on May 17, 2023.
 

sdk/version

@@ -1 +1 @@
-0.3.3
+0.3.4

src/dockerfiles/Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL = all
-VERSION = 0.3.3
+VERSION = 0.3.4
 
 # By default, turn off the Makefile practice of printing each command before
 # you run it.

src/dockerfiles/connectors/athena.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/connectors/bigquery.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/connectors/mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@spiralai.co> version: 0.1
 

src/dockerfiles/connectors/postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/connectors/s3.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/connectors/snowflake.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/connectors/sqlserver.dockerfile

@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.3.3
+FROM aqueducthq/base_connector:0.3.4
 
 MAINTAINER Aqueduct <hello@aqueducthq.com> version: 0.1
 

src/dockerfiles/function/function310.dockerfile

@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4
 
 ENV PYTHONUNBUFFERED 1
 

src/dockerfiles/function/function37.dockerfile

@@ -16,7 +16,7 @@ boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
 typing_extensions==4.3.0 \
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4
 
 
 ENV PYTHONUNBUFFERED 1

src/dockerfiles/function/function38.dockerfile

@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4
 
 ENV PYTHONUNBUFFERED 1
 

src/dockerfiles/function/function39.dockerfile

@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4
 
 ENV PYTHONUNBUFFERED 1
 

src/dockerfiles/gpu/py310_env.yml

@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.3.3
+    - aqueduct-ml==0.3.4

src/dockerfiles/gpu/py37_env.yml

@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.3.3
+    - aqueduct-ml==0.3.4

src/dockerfiles/gpu/py38_env.yml

@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.3.3
+    - aqueduct-ml==0.3.4

src/dockerfiles/gpu/py39_env.yml

@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.3.3
+    - aqueduct-ml==0.3.4

src/dockerfiles/lambda/function/requirements-37.txt

@@ -7,4 +7,4 @@ boto3==1.18.0
 pydantic==1.9.0
 scikit_learn==1.0.2
 typing_extensions==4.3.0
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4

src/dockerfiles/lambda/function/requirements.txt

@@ -6,4 +6,4 @@ pyarrow==7.0.0
 boto3==1.18.0
 pydantic==1.9.0
 scikit_learn==1.0.2
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4

src/dockerfiles/lambda/requirements.txt

@@ -6,4 +6,4 @@ pydantic==1.9.0
 pyyaml
 SQLAlchemy==1.4.30
 typing_extensions==4.3.0
-aqueduct-ml==0.3.3
+aqueduct-ml==0.3.4

src/dockerfiles/spark/spark-py310-env.dockerfile

@@ -25,7 +25,7 @@ RUN	wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws
     unzip awscliv2.zip && ./aws/install 
 
 RUN conda activate py310_env
-RUN pip install conda-pack aqueduct-ml==0.3.3
+RUN pip install conda-pack aqueduct-ml==0.3.4
 
 COPY ./spark/create-conda-env.sh /
 

src/dockerfiles/spark/spark-py37-env.dockerfile

@@ -25,7 +25,7 @@ RUN	wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws
     unzip awscliv2.zip && ./aws/install 
 
 RUN conda activate py37_env
-RUN pip install conda-pack aqueduct-ml==0.3.3
+RUN pip install conda-pack aqueduct-ml==0.3.4
 
 COPY ./spark/create-conda-env.sh /
 

src/dockerfiles/spark/spark-py38-env.dockerfile

@@ -25,7 +25,7 @@ RUN	wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws
     unzip awscliv2.zip && ./aws/install 
 
 RUN conda activate py38_env
-RUN pip install conda-pack aqueduct-ml==0.3.3
+RUN pip install conda-pack aqueduct-ml==0.3.4
 
 COPY ./spark/create-conda-env.sh /
 

src/dockerfiles/spark/spark-py39-env.dockerfile

@@ -25,7 +25,7 @@ RUN	wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O aws
     unzip awscliv2.zip && ./aws/install 
 
 RUN conda activate py39_env
-RUN pip install conda-pack aqueduct-ml==0.3.3
+RUN pip install conda-pack aqueduct-ml==0.3.4
 
 COPY ./spark/create-conda-env.sh /
 

src/golang/lib/constants.go

@@ -1,3 +1,3 @@
 package lib
 
-const ServerVersionNumber = "0.3.3"
+const ServerVersionNumber = "0.3.4"

src/python/bin/aqueduct

@@ -38,7 +38,7 @@ base_directory = os.path.join(os.environ["HOME"], ".aqueduct")
 server_directory = os.path.join(os.environ["HOME"], ".aqueduct", "server")
 ui_directory = os.path.join(os.environ["HOME"], ".aqueduct", "ui")
 
-package_version = "0.3.3"
+package_version = "0.3.4"
 aws_credentials_path = os.path.join(os.environ["HOME"], ".aws", "credentials")
 
 default_server_port = 8080

src/python/version

@@ -1 +1 @@
-0.3.3
+0.3.4

src/ui/app/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@aqueducthq/ui",
   "author": "Aqueduct, Inc. <hello@aqueducthq.com>",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "scripts": {
     "start": "parcel --no-cache index.html",
     "build": "parcel build --public-url /dist --dist-dir dist/default index.html",
     "lint": "eslint '*/**/*.{js,ts,tsx}' --format table",
     "lint:fix": "eslint '*/**/*.{js,ts,tsx}' --format table --fix"
   },
   "dependencies": {
-    "@aqueducthq/common": "0.3.3",
+    "@aqueducthq/common": "0.3.4",
     "@emotion/react": "^11.10.5",
     "@emotion/styled": "^11.10.5",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",

src/ui/common/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aqueducthq/common",
   "author": "Aqueduct <hello@aqueducthq.com>",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "alias": {

src/ui/common/src/components/pages/data/index.tsx

@@ -12,9 +12,7 @@ import getPathPrefix from '../../../utils/getPathPrefix';
 import { CheckLevel } from '../../../utils/operators';
 import ExecutionStatus from '../../../utils/shared';
 import DefaultLayout from '../../layouts/default';
-import PaginatedSearchTable, {
-  PaginatedSearchTableData,
-} from '../../tables/PaginatedSearchTable';
+import PaginatedSearchTable from '../../tables/PaginatedSearchTable';
 import { LayoutProps } from '../types';
 import CheckItem from '../workflows/components/CheckItem';
 import ExecutionStatusLink from '../workflows/components/ExecutionStatusLink';
@@ -50,38 +48,27 @@ const DataPage: React.FC<Props> = ({ user, Layout = DefaultLayout }) => {
   }, []);
 
   const onGetColumnValue = (row, column) => {
-    let value = row[column.name];
-
-    switch (column.name) {
+    const value = row[column];
+    switch (column) {
       case 'workflow':
       case 'name':
         const { name, url, status } = value;
-        value = <ExecutionStatusLink name={name} url={url} status={status} />;
-        break;
+        return <ExecutionStatusLink name={name} url={url} status={status} />;
       case 'created_at':
-        value = row[column.name].toLocaleString();
-        break;
+        return value.toLocaleString();
       case 'metrics': {
-        value = <MetricItem metrics={value} />;
-        break;
+        return <MetricItem metrics={value} />;
       }
       case 'checks': {
-        value = <CheckItem checks={value} />;
-        break;
+        return <CheckItem checks={value} />;
       }
       case 'type': {
-        value = (
-          <Typography fontFamily="monospace">{row[column.name]}</Typography>
-        );
-        break;
+        return <Typography fontFamily="monospace">{value}</Typography>;
       }
       default: {
-        value = row[column.name];
-        break;
+        return value;
       }
     }
-
-    return value;
   };
 
   let tableData = [];
@@ -184,21 +171,6 @@ const DataPage: React.FC<Props> = ({ user, Layout = DefaultLayout }) => {
     },
   ];
 
-  const artifactList: PaginatedSearchTableData = {
-    schema: {
-      fields: [
-        { name: 'name', type: 'varchar' },
-        { name: 'created_at', displayName: 'Created At', type: 'varchar' },
-        { name: 'workflow', type: 'varchar' },
-        { name: 'type', type: 'varchar' },
-        { name: 'metrics', type: 'varchar' },
-        { name: 'checks', type: 'varchar' },
-      ],
-      pandas_version: '1.5.1',
-    },
-    data: tableData,
-  };
-
   const noItemsMessage = (
     <Typography variant="h5">
       There are no data artifacts created yet. Create one right by running a
@@ -252,9 +224,17 @@ const DataPage: React.FC<Props> = ({ user, Layout = DefaultLayout }) => {
       breadcrumbs={[BreadcrumbLink.HOME, BreadcrumbLink.DATA]}
       user={user}
     >
-      {artifactList.data?.length && artifactList.data?.length > 0 ? (
+      {tableData?.length && tableData?.length > 0 ? (
         <PaginatedSearchTable
-          data={artifactList}
+          data={tableData}
+          columns={[
+            'name',
+            'created_at',
+            'workflow',
+            'type',
+            'metrics',
+            'checks',
+          ]}
           searchEnabled={true}
           onGetColumnValue={onGetColumnValue}
           onChangeRowsPerPage={onChangeRowsPerPage}