Merge pull request #1384 from keboola/ct-1688-add-chaned-parameter-lo…

…ad-to-ws test changeSince/until ws load
keboola · Aug 13, 2024 · 99eb896 · 99eb896
2 parents f872b33 + 00271a1
commit 99eb896
Show file tree

Hide file tree

Showing 2 changed files with 133 additions and 0 deletions.
diff --git a/apiary.apib b/apiary.apib
@@ -5139,6 +5139,10 @@ Loads tables from Storage into a Workspace. Not supported for Teradata. BigQuery
                     + rows (optional, number) - Limits the number of returned rows
                     + days (optional, number) - `DEPRECATED will be removed in the future.` Return rows created or updated in the last X days.
                     + seconds (optional, number) - Returns rows created or updated in the last X seconds.
+                    + changedSince (optional) - Currently only supported for Snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php).
+                        **If the `days` or `seconds` parameter is set, this value is ignored.**
+                    + changedUntil (optional) - Currently only supported for snowflake - Filtering by import date - timestamp of import is stored within each row. Both until and since values can be a unix timestamp or any date accepted by [strtotime](http://www.php.net/manual/en/function.strtotime.php).
+                        **If the `days` or `seconds` parameter is set, this vallue are ignored.** Examples: `-2 days`, `1360138863`, `2013-02-12T15:19:21+00:00`
                     + columns[] (optional, array) - Array of column definition; by default all columns are exported.
                         + Items
                             + (object)

diff --git a/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php b/tests/Backend/Snowflake/TypedTableWorkspacesLoadTest.php
@@ -2,7 +2,9 @@
 
 namespace Keboola\Test\Backend\Snowflake;
 
+use Generator;
 use Keboola\Csv\CsvFile;
+use Keboola\StorageApi\Client;
 use Keboola\StorageApi\ClientException;
 use Keboola\StorageApi\Workspaces;
 use Keboola\TableBackendUtils\Column\ColumnInterface;
@@ -406,6 +408,133 @@ public function testRowsParameterInvalidType(): void
         $this->assertEquals(2, $numrows, 'rows parameter');
     }
 
+    /**
+     * @dataProvider wrongChangedDataProvider
+     */
+    public function testWorkspaceLoadFilterByChangedWithWrongValues(array $changed): void
+    {
+        $tableId = $this->createTableUsers();
+        $workspaces = new Workspaces($this->workspaceSapiClient);
+        $workspace = $this->initTestWorkspace();
+
+        $options = [
+            'input' => [
+                array_merge(
+                    [
+                        'source' => $tableId,
+                        'destination' => 'filter-test',
+                    ],
+                    $changed,
+                ),
+            ],
+        ];
+
+        try {
+            $workspaces->loadWorkspaceData($workspace['id'], $options);
+            $this->fail('Trying to filter with wrong values should fail');
+        } catch (ClientException $e) {
+            $this->assertStringContainsString(sprintf('Invalid "%s" parameter', array_keys($changed)[0]), $e->getMessage());
+        }
+    }
+
+    public function wrongChangedDataProvider(): Generator
+    {
+        yield 'wrong changedSince' => [
+            ['changedSince' => 'roman'],
+        ];
+
+        yield 'wrong changedUntil' => [
+            ['changedUntil' => 'roman'],
+        ];
+    }
+
+    /**
+     * @dataProvider sinceDataProvider
+     */
+    public function testWorkspaceLoadFilterByChanged(?string $changedSince, ?string $changedUntil, int $expectedResult): void
+    {
+        // initial import
+        $tableId = $this->createTableUsers();
+        $timesOfRun = [];
+        $timesOfRun['tableCreated'] = time();
+        // first import
+        $this->_client->writeTableAsync(
+            $tableId,
+            new CsvFile(__DIR__ . '/../../_data/users.csv'),
+            [
+                'incremental' => true,
+            ],
+        );
+        $timesOfRun['fistImportTime'] = time();
+
+        // second import
+        $this->_client->writeTableAsync(
+            $tableId,
+            new CsvFile(__DIR__ . '/../../_data/users.csv'),
+            [
+                'incremental' => true,
+            ],
+        );
+        $timesOfRun['secondImportTime'] = time();
+
+        $options = [
+            'input' => [
+                [
+                    'source' => $tableId,
+                    'destination' => 'filter-test',
+                    'changedSince' => $timesOfRun[$changedSince] ?? null,
+                    'changedUntil' => $timesOfRun[$changedUntil] ?? null,
+                ],
+            ],
+        ];
+
+        $data = $this->_client->getTableDataPreview($tableId);
+        $this->assertEquals(15, count(Client::parseCsv($data)));
+
+        $workspaces = new Workspaces($this->workspaceSapiClient);
+        $workspace = $this->initTestWorkspace();
+        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);
+
+        $workspaces->loadWorkspaceData($workspace['id'], $options);
+
+        $data = $backend->fetchAll('filter-test');
+
+        $this->assertCount($expectedResult, $data);
+    }
+
+    public function sinceDataProvider(): Generator
+    {
+        yield 'since: tableCreated; until: null' => [
+            'changedSince' => 'tableCreated',
+            'changedUntil' => null,
+            'expectedResult' => 10,
+        ];
+
+        yield 'since: tableCreated; until: fistImportTime' => [
+            'changedSince' => 'tableCreated',
+            'changedUntil' => 'fistImportTime',
+            'expectedResult' => 5,
+        ];
+
+        yield 'since: null; until: fistImportTime' => [
+            'changedSince' => null,
+            'changedUntil' => 'fistImportTime',
+            'expectedResult' => 10,
+        ];
+
+        yield 'since: fistImportTime; until: secondImportTime' => [
+            'changedSince' => 'fistImportTime',
+            'changedUntil' => 'secondImportTime',
+            'expectedResult' => 5,
+        ];
+
+        yield 'all' => [
+            'changedSince' => null,
+            'changedUntil' => null,
+            'expectedResult' => 15,
+        ];
+    }
+
     /**
      * @param array<mixed> $exportOptions
      * @param array<mixed> $expectedResult