Run tests for postgres-scanner on CI (#3141)

kuzudb · Mar 26, 2024 · 161f2f2 · 161f2f2
1 parent e2567c3
commit 161f2f2
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 33 deletions.
diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
@@ -46,6 +46,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update PostgreSQL host
+        working-directory: extension/postgres_scanner/test/test_files
+        env:
+          FNAME: postgres_scanner.test
+          FIND: "localhost"
+        run: |
+          node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});'
+          cat postgres_scanner.test
+
       - name: Ensure Python dependencies
         run: |
           pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu
@@ -179,6 +188,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update PostgreSQL host
+        working-directory: extension/postgres_scanner/test/test_files
+        env:
+          FNAME: postgres_scanner.test
+          FIND: "localhost"
+        run: |
+          node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});'
+          cat postgres_scanner.test
+
       - name: Ensure Python dependencies
         run: |
           pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu
@@ -225,10 +243,20 @@ jobs:
       AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
+      PG_HOST: ${{ secrets.PG_HOST }}
       RUN_ID: "$(hostname)-$([Math]::Floor((Get-Date).TimeOfDay.TotalSeconds))"
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update PostgreSQL host
+        working-directory: extension/postgres_scanner/test/test_files
+        env:
+          FNAME: postgres_scanner.test
+          FIND: "localhost"
+        run: |
+          node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});'
+          cat postgres_scanner.test
+
       - name: Ensure Python dependencies
         run: |
           pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu
@@ -430,10 +458,20 @@ jobs:
       AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
+      PG_HOST: ${{ secrets.PG_HOST }}
       RUN_ID: "$(hostname)-$(date +%s)"
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update PostgreSQL host
+        working-directory: extension/postgres_scanner/test/test_files
+        env:
+          FNAME: postgres_scanner.test
+          FIND: "localhost"
+        run: |
+          node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});'
+          cat postgres_scanner.test
+
       - name: Ensure Python dependencies
         run: |
           pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu

diff --git a/Makefile b/Makefile
@@ -159,12 +159,8 @@ extension-test:
 		-DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \
 		-DBUILD_EXTENSION_TESTS=TRUE \
 	)
-	dropdb -h localhost -p 5432 -U ci pgscan || true
-	createdb -h localhost -p 5432 -U ci -E UTF8 -T template0  -e pgscan
-	psql -U ci -d pgscan -f extension/postgres_scanner/test/test_files/create_test_db.sql
 	ctest --test-dir build/release/extension --output-on-failure -j ${TEST_JOBS}
 	aws s3 rm s3://kuzu-dataset-us/${RUN_ID}/ --recursive
-	dropdb -h localhost -p 5432 -U ci pgscan
 
 extension-debug:
 	$(call run-cmake-debug, \

diff --git a/extension/postgres_scanner/test/test_files/create_test_db.sql b/extension/postgres_scanner/test/test_files/create_test_db.sql
@@ -17,7 +17,7 @@ SET client_min_messages = warning;
 SET row_security = off;
 
 --
--- Name: audience_type; Type: TYPE; Schema: public; Owner: z473chen
+-- Name: audience_type; Type: TYPE; Schema: public; Owner: ci
 --
 
 CREATE TYPE public.audience_type AS (
@@ -26,10 +26,10 @@ CREATE TYPE public.audience_type AS (
 );
 
 
-ALTER TYPE public.audience_type OWNER TO z473chen;
+ALTER TYPE public.audience_type OWNER TO ci;
 
 --
--- Name: description_type; Type: TYPE; Schema: public; Owner: z473chen
+-- Name: description_type; Type: TYPE; Schema: public; Owner: ci
 --
 
 CREATE TYPE public.description_type AS (
@@ -50,10 +50,10 @@ CREATE TYPE public.description_type AS (
 );
 
 
-ALTER TYPE public.description_type OWNER TO z473chen;
+ALTER TYPE public.description_type OWNER TO ci;
 
 --
--- Name: mood; Type: TYPE; Schema: public; Owner: z473chen
+-- Name: mood; Type: TYPE; Schema: public; Owner: ci
 --
 
 CREATE TYPE public.mood AS ENUM (
@@ -63,10 +63,10 @@ CREATE TYPE public.mood AS ENUM (
 );
 
 
-ALTER TYPE public.mood OWNER TO z473chen;
+ALTER TYPE public.mood OWNER TO ci;
 
 --
--- Name: state_type; Type: TYPE; Schema: public; Owner: z473chen
+-- Name: state_type; Type: TYPE; Schema: public; Owner: ci
 --
 
 CREATE TYPE public.state_type AS (
@@ -75,10 +75,10 @@ CREATE TYPE public.state_type AS (
 );
 
 
-ALTER TYPE public.state_type OWNER TO z473chen;
+ALTER TYPE public.state_type OWNER TO ci;
 
 --
--- Name: stock_type; Type: TYPE; Schema: public; Owner: z473chen
+-- Name: stock_type; Type: TYPE; Schema: public; Owner: ci
 --
 
 CREATE TYPE public.stock_type AS (
@@ -87,14 +87,14 @@ CREATE TYPE public.stock_type AS (
 );
 
 
-ALTER TYPE public.stock_type OWNER TO z473chen;
+ALTER TYPE public.stock_type OWNER TO ci;
 
 SET default_tablespace = '';
 
 SET default_table_access_method = heap;
 
 --
--- Name: movies; Type: TABLE; Schema: public; Owner: z473chen
+-- Name: movies; Type: TABLE; Schema: public; Owner: ci
 --
 
 CREATE TABLE public.movies (
@@ -107,10 +107,10 @@ CREATE TABLE public.movies (
 );
 
 
-ALTER TABLE public.movies OWNER TO z473chen;
+ALTER TABLE public.movies OWNER TO ci;
 
 --
--- Name: organisation; Type: TABLE; Schema: public; Owner: z473chen
+-- Name: organisation; Type: TABLE; Schema: public; Owner: ci
 --
 
 CREATE TABLE public.organisation (
@@ -128,10 +128,10 @@ CREATE TABLE public.organisation (
 );
 
 
-ALTER TABLE public.organisation OWNER TO z473chen;
+ALTER TABLE public.organisation OWNER TO ci;
 
 --
--- Name: person; Type: TABLE; Schema: public; Owner: z473chen
+-- Name: person; Type: TABLE; Schema: public; Owner: ci
 --
 
 CREATE TABLE public.person (
@@ -152,21 +152,21 @@ CREATE TABLE public.person (
 );
 
 
-ALTER TABLE public.person OWNER TO z473chen;
+ALTER TABLE public.person OWNER TO ci;
 
 --
--- Name: persontest; Type: TABLE; Schema: public; Owner: z473chen
+-- Name: persontest; Type: TABLE; Schema: public; Owner: ci
 --
 
 CREATE TABLE public.persontest (
     id integer
 );
 
 
-ALTER TABLE public.persontest OWNER TO z473chen;
+ALTER TABLE public.persontest OWNER TO ci;
 
 --
--- Data for Name: movies; Type: TABLE DATA; Schema: public; Owner: z473chen
+-- Data for Name: movies; Type: TABLE DATA; Schema: public; Owner: ci
 --
 
 COPY public.movies (name, length, note, description, content, audience) FROM stdin;
@@ -177,7 +177,7 @@ Roma	298	the movie is very interesting and funny	(1223,100,10003,"2011-02-11 16:
 
 
 --
--- Data for Name: organisation; Type: TABLE DATA; Schema: public; Owner: z473chen
+-- Data for Name: organisation; Type: TABLE DATA; Schema: public; Owner: ci
 --
 
 COPY public.organisation (id, name, orgcode, mark, score, history, licensevalidinterval, rating, state, stock, info) FROM stdin;
@@ -188,7 +188,7 @@ COPY public.organisation (id, name, orgcode, mark, score, history, licensevalidi
 
 
 --
--- Data for Name: person; Type: TABLE DATA; Schema: public; Owner: z473chen
+-- Data for Name: person; Type: TABLE DATA; Schema: public; Owner: ci
 --
 
 COPY public.person (id, fname, gender, isstudent, isworker, age, eyesight, birthdate, registertime, lastjobduration, workedhours, usednames, height, u) FROM stdin;
@@ -204,31 +204,31 @@ COPY public.person (id, fname, gender, isstudent, isworker, age, eyesight, birth
 
 
 --
--- Data for Name: persontest; Type: TABLE DATA; Schema: public; Owner: z473chen
+-- Data for Name: persontest; Type: TABLE DATA; Schema: public; Owner: ci
 --
 
 COPY public.persontest (id) FROM stdin;
 \.
 
 
 --
--- Name: movies movies_pkey; Type: CONSTRAINT; Schema: public; Owner: z473chen
+-- Name: movies movies_pkey; Type: CONSTRAINT; Schema: public; Owner: ci
 --
 
 ALTER TABLE ONLY public.movies
     ADD CONSTRAINT movies_pkey PRIMARY KEY (name);
 
 
 --
--- Name: organisation organisation_pkey; Type: CONSTRAINT; Schema: public; Owner: z473chen
+-- Name: organisation organisation_pkey; Type: CONSTRAINT; Schema: public; Owner: ci
 --
 
 ALTER TABLE ONLY public.organisation
     ADD CONSTRAINT organisation_pkey PRIMARY KEY (id);
 
 
 --
--- Name: person person_pkey; Type: CONSTRAINT; Schema: public; Owner: z473chen
+-- Name: person person_pkey; Type: CONSTRAINT; Schema: public; Owner: ci
 --
 
 ALTER TABLE ONLY public.person

diff --git a/extension/postgres_scanner/test/test_files/postgres_scanner.test b/extension/postgres_scanner/test/test_files/postgres_scanner.test
@@ -6,7 +6,7 @@
 -CASE ScanPostgresTable
 -STATEMENT load extension "${KUZU_ROOT_DIRECTORY}/extension/postgres_scanner/build/libpostgres_scanner.kuzu_extension"
 ---- ok
--STATEMENT ATTACH 'dbname=pgscan user=ci host=127.0.0.1' as tinysnb (dbtype 'POSTGRES');
+-STATEMENT ATTACH 'dbname=pgscan user=ci host=localhost' as tinysnb (dbtype 'POSTGRES');
 ---- ok
 -STATEMENT LOAD FROM tinysnb_person RETURN *;
 ---- 8
@@ -34,7 +34,7 @@ Catalog exception: Table: person1 does not exist.
 -STATEMENT LOAD FROM tinysnb1_person RETURN *;
 ---- error
 Binder exception: No database named tinysnb1 has been attached.
--STATEMENT ATTACH 'dbname=pgscan user=ci host=127.0.0.1' (dbtype 'POSTGRES');
+-STATEMENT ATTACH 'dbname=pgscan user=ci host=localhost' (dbtype 'POSTGRES');
 ---- ok
 -STATEMENT LOAD FROM pgscan_movies RETURN count(*);
 ---- 1
@@ -44,5 +44,6 @@ Binder exception: No database named tinysnb1 has been attached.
 The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie
 -LOG IncorrectConnectionStr
 -STATEMENT ATTACH 'dbname=test2132131 user=ci host=127.0.0.1' as tinysnb (dbtype 'POSTGRES');
----- error
-Binder exception: Failed to attach postgres database due to: IO Error: Unable to connect to Postgres at dbname=test2132131 user=ci host=127.0.0.1: connection to server at "127.0.0.1", port 5432 failed: FATAL:  database "test2132131" does not exist
+---- error 2 rows
+Binder exception: Failed to attach postgres database due to: IO Error: Unable to connect to Postgres at dbname=test2132131 user=ci host=127.0.0.1: connection to server at "127.0.0.1", port 5432 failed: Connection refused
+	Is the server running on that host and accepting TCP/IP connections?
diff --git a/extension/test_list b/extension/test_list
@@ -0,0 +1,13 @@
+PostgresScanner.ScanPostgresTable /Users/lc/Developer/kuzu/extension/postgres_scanner/test/test_files/postgres_scanner.test
+DuckDBScanner.InvalidDuckDBDatabase /Users/lc/Developer/kuzu/extension/duckdb_scanner/test/test_files/duckdb_scanner.test
+DuckDBScanner.ScanDuckDBTable /Users/lc/Developer/kuzu/extension/duckdb_scanner/test/test_files/duckdb_scanner.test
+S3.DISABLED_GlobS3Folder /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_glob.test
+S3.DISABLED_UploadToS3 /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_upload.test
+S3.DISABLED_ScanFromDifferentAWSS3File /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_download.test
+S3.ScanFromUWS3File /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_download.test
+S3.DuplicateOption /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_download.test
+S3.S3Option /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/s3_download.test
+HTTPFS.ScanFromHTTPS /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/http.test
+HTTPFS.CopyFromHTTPCSV /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/http.test
+HTTPFS.ScanFromLargeFiles /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/http.test
+HTTPFS.ScanFromHTTPCSV /Users/lc/Developer/kuzu/extension/httpfs/test/test_files/http.test