From 99e7acd4819442ee9e978c817360ad0ffeda1854 Mon Sep 17 00:00:00 2001 From: Roman Bracinik Date: Thu, 8 Aug 2024 15:53:17 +0200 Subject: [PATCH 1/5] Init tests --- .../TypedTableWorkspacesLoadTest.php | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php index 29a67ed08..9b98bbd79 100644 --- a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php +++ b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php @@ -2,7 +2,9 @@ namespace Keboola\Test\Backend\Snowflake; +use Generator; use Keboola\Csv\CsvFile; +use Keboola\StorageApi\Client; use Keboola\StorageApi\ClientException; use Keboola\StorageApi\Workspaces; use Keboola\TableBackendUtils\Column\ColumnInterface; @@ -406,6 +408,93 @@ public function testRowsParameterInvalidType(): void $this->assertEquals(2, $numrows, 'rows parameter'); } + /** + * @dataProvider sinceDataProvider + */ + public function testWorkspaceLoadFilterByChanged(?string $changedSince, ?string $changedUntil, int $expectedResult): void + { + // initial import + $tableId = $this->createTableUsers(); + $timesOfRun = []; + $timesOfRun['tableCreated'] = time(); + // first import + $this->_client->writeTableAsync( + $tableId, + new CsvFile(__DIR__ . '/../../_data/users.csv'), + [ + 'incremental' => true, + ], + ); + $timesOfRun['fistImportTime'] = time(); + + // second import + $this->_client->writeTableAsync( + $tableId, + new CsvFile(__DIR__ . '/../../_data/users.csv'), + [ + 'incremental' => true, + ], + ); + $timesOfRun['secondImportTime'] = time(); + + $options = [ + 'input' => [ + [ + 'source' => $tableId, + 'destination' => 'filter-test', + 'changedSince' => $timesOfRun[$changedSince] ?? null, + 'changedUntil' => $timesOfRun[$changedUntil] ?? null, + ], + ], + ]; + + $data = $this->_client->getTableDataPreview($tableId); + $this->assertEquals(15, count(Client::parseCsv($data))); + + $workspaces = new Workspaces($this->workspaceSapiClient); + $workspace = $this->initTestWorkspace(); + $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace); + + $workspaces->loadWorkspaceData($workspace['id'], $options); + + $data = $backend->fetchAll('filter-test'); + + $this->assertCount($expectedResult, $data); + } + + public function sinceDataProvider(): Generator + { + yield 'since: tableCreated; until: null' => [ + 'changedSince' => 'tableCreated', + 'changedUntil' => null, + 'expectedResult' => 10, + ]; + + yield 'since: tableCreated; until: fistImportTime' => [ + 'changedSince' => 'tableCreated', + 'changedUntil' => 'fistImportTime', + 'expectedResult' => 5, + ]; + + yield 'since: null; until: fistImportTime' => [ + 'changedSince' => null, + 'changedUntil' => 'fistImportTime', + 'expectedResult' => 10, + ]; + + yield 'since: fistImportTime; until: secondImportTime' => [ + 'changedSince' => 'fistImportTime', + 'changedUntil' => 'secondImportTime', + 'expectedResult' => 5, + ]; + + yield 'all' => [ + 'changedSince' => null, + 'changedUntil' => null, + 'expectedResult' => 15, + ]; + } + /** * @param array $exportOptions * @param array $expectedResult From 0b92e329b1b955cb4f54e1af8bd1d7f5e6dec2a0 Mon Sep 17 00:00:00 2001 From: Roman Bracinik Date: Fri, 9 Aug 2024 10:35:33 +0200 Subject: [PATCH 2/5] update doc --- apiary.apib | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apiary.apib b/apiary.apib index 7aa9daa2e..9d396e384 100644 --- a/apiary.apib +++ b/apiary.apib @@ -5139,6 +5139,10 @@ Loads tables from Storage into a Workspace. Not supported for Teradata. BigQuery + rows (optional, number) - Limits the number of returned rows + days (optional, number) - `DEPRECATED will be removed in the future.` Return rows created or updated in the last X days. + seconds (optional, number) - Returns rows created or updated in the last X seconds. + + changedSince (optional) - Currently only supported for snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php). + **If the `days` or `seconds` parameter is set these values are ignored.** + + changedUntil (optional) - Currently only supported for snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php). + **If the `days` or `seconds` parameter is set these values are ignored.** Examples: `-2 days`, `1360138863`, `2013-02-12T15:19:21+00:00` + columns[] (optional, array) - Array of column definition; by default all columns are exported. + Items + (object) From 10c8da965b1a21c5027dba064d56d2ed773e0095 Mon Sep 17 00:00:00 2001 From: Roman Bracinik Date: Mon, 12 Aug 2024 09:52:51 +0200 Subject: [PATCH 3/5] test wrong values --- .../TypedTableWorkspacesLoadTest.php | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php index 9b98bbd79..d5105275a 100644 --- a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php +++ b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php @@ -408,6 +408,46 @@ public function testRowsParameterInvalidType(): void $this->assertEquals(2, $numrows, 'rows parameter'); } + /** + * @dataProvider wrongChangedDataProvider + */ + public function testWorkspaceLoadFilterByChangedWithWrongValues(array $changed): void + { + $tableId = $this->createTableUsers(); + $workspaces = new Workspaces($this->workspaceSapiClient); + $workspace = $this->initTestWorkspace(); + + $options = [ + 'input' => [ + array_merge( + [ + 'source' => $tableId, + 'destination' => 'filter-test', + ], + $changed + ), + ], + ]; + + try { + $workspaces->loadWorkspaceData($workspace['id'], $options); + $this->fail('Trying to filter with wrong values should fail'); + } catch (ClientException $e) { + $this->assertStringContainsString(sprintf('Invalid "%s" parameter', array_keys($changed)[0]), $e->getMessage()); + } + } + + public function wrongChangedDataProvider(): Generator + { + yield 'wrong changedSince' => [ + ['changedSince' => 'roman',] + ]; + + yield 'wrong changedUntil' => [ + ['changedUntil' => 'roman',] + ]; + } + /** * @dataProvider sinceDataProvider */ From f80283b41ccbd327e49764b869ac279338c4b789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Bracin=C3=ADk?= Date: Mon, 12 Aug 2024 09:55:15 +0200 Subject: [PATCH 4/5] Update apiary.apib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jiří Semmler <13363655+jirkasemmler@users.noreply.github.com> --- apiary.apib | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apiary.apib b/apiary.apib index 9d396e384..d0e1685db 100644 --- a/apiary.apib +++ b/apiary.apib @@ -5139,10 +5139,10 @@ Loads tables from Storage into a Workspace. Not supported for Teradata. BigQuery + rows (optional, number) - Limits the number of returned rows + days (optional, number) - `DEPRECATED will be removed in the future.` Return rows created or updated in the last X days. + seconds (optional, number) - Returns rows created or updated in the last X seconds. - + changedSince (optional) - Currently only supported for snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php). - **If the `days` or `seconds` parameter is set these values are ignored.** + + changedSince (optional) - Currently only supported for Snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php). + **If the `days` or `seconds` parameter is set, this value is ignored.** + changedUntil (optional) - Currently only supported for snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php). - **If the `days` or `seconds` parameter is set these values are ignored.** Examples: `-2 days`, `1360138863`, `2013-02-12T15:19:21+00:00` + **If the `days` or `seconds` parameter is set, this vallue are ignored.** Examples: `-2 days`, `1360138863`, `2013-02-12T15:19:21+00:00` + columns[] (optional, array) - Array of column definition; by default all columns are exported. + Items + (object) From 00271a1894f6515429d96d9cf1686ad98ef57516 Mon Sep 17 00:00:00 2001 From: Roman Bracinik Date: Mon, 12 Aug 2024 10:01:18 +0200 Subject: [PATCH 5/5] fix cs --- tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php index d5105275a..31ae9d308 100644 --- a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php +++ b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php @@ -424,7 +424,7 @@ public function testWorkspaceLoadFilterByChangedWithWrongValues(array $changed): 'source' => $tableId, 'destination' => 'filter-test', ], - $changed + $changed, ), ], ]; @@ -440,11 +440,11 @@ public function testWorkspaceLoadFilterByChangedWithWrongValues(array $changed): public function wrongChangedDataProvider(): Generator { yield 'wrong changedSince' => [ - ['changedSince' => 'roman',] + ['changedSince' => 'roman'], ]; yield 'wrong changedUntil' => [ - ['changedUntil' => 'roman',] + ['changedUntil' => 'roman'], ]; }