Merge pull request #51 from keboola/ondra-check-table-restorable

Check if table is restorable
keboola · Sep 20, 2024 · 79f294c · 79f294c
2 parents a85f312 + d441c7f
commit 79f294c
Show file tree

Hide file tree

Showing 8 changed files with 192 additions and 0 deletions.
diff --git a/src/Restore.php b/src/Restore.php
@@ -499,6 +499,7 @@ function ($bucket) {
             if ($tableInfo['isAlias'] === true) {
                 continue;
             }
+            $this->checkTableRestorable($tableInfo);
 
             $tableId = $tableInfo['id'];
             $bucketId = $tableInfo['bucket']['id'];
@@ -890,4 +891,33 @@ public function restoreNotifications(): void
             ));
         }
     }
+
+    private function checkTableRestorable(array $tableInfo): void
+    {
+        // check if primage key has nullable column
+        if (($tableInfo['isTyped'] ?? false) === true) {
+            foreach ($tableInfo['columnMetadata'] as $columnName => $columnMetadata) {
+                if (!in_array($columnName, $tableInfo['primaryKey'], true)) {
+                    continue;
+                }
+                $columnMetadata = array_filter(
+                    $columnMetadata,
+                    fn($v) => $v['provider'] === 'storage',
+                );
+
+                $columnMetadataList = array_combine(
+                    array_map(fn($v) => $v['key'], $columnMetadata),
+                    array_map(fn($v) => $v['value'], $columnMetadata),
+                );
+                if (isset($columnMetadataList['KBC.datatype.nullable']) &&
+                    $columnMetadataList['KBC.datatype.nullable'] === '1') {
+                    $this->logger->warning(sprintf(
+                        'Table "%s" cannot be restored because the primary key column "%s" is nullable.',
+                        $tableInfo['name'],
+                        $columnName,
+                    ));
+                }
+            }
+        }
+    }
 }
diff --git a/tests/AbsRestoreTest.php b/tests/AbsRestoreTest.php
@@ -1051,4 +1051,23 @@ public function testRestoreNotifications(): void
             (string) json_encode($restoreNotifications, JSON_PRETTY_PRINT),
         );
     }
+
+    public function testRestoreTableWithNullablePKs(): void
+    {
+        $logger = new TestLogger();
+        $restore = new AbsRestore(
+            $this->sapiClient,
+            $this->absClient,
+            getenv('TEST_AZURE_CONTAINER_NAME') . '-table-with-nullable-pk',
+            $logger,
+        );
+        $restore->setDryRunMode();
+
+        $restore->restoreBuckets();
+        $restore->restoreTables();
+
+        self::assertTrue($logger->hasWarning(
+            'Table "firstTable" cannot be restored because the primary key column "Id" is nullable.',
+        ));
+    }
 }
diff --git a/tests/S3RestoreTest.php b/tests/S3RestoreTest.php
@@ -1090,4 +1090,24 @@ public function testRestoreNotifications(): void
             (string) json_encode($restoreNotifications, JSON_PRETTY_PRINT),
         );
     }
+
+    public function testRestoreTableWithNullablePKs(): void
+    {
+        $logger = new TestLogger();
+        $restore = new S3Restore(
+            $this->sapiClient,
+            $this->s3Client,
+            (string) getenv('TEST_AWS_S3_BUCKET'),
+            'table-with-nullable-pk',
+            $logger,
+        );
+        $restore->setDryRunMode();
+
+        $restore->restoreBuckets();
+        $restore->restoreTables();
+
+        self::assertTrue($logger->hasWarning(
+            'Table "firstTable" cannot be restored because the primary key column "Id" is nullable.',
+        ));
+    }
 }
diff --git a/tests/data/backups/table-with-nullable-pk/buckets.json b/tests/data/backups/table-with-nullable-pk/buckets.json
@@ -0,0 +1,18 @@
+[
+  {
+    "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/buckets\/in.c-bucket",
+    "id": "in.c-bucket",
+    "name": "c-bucket",
+    "stage": "in",
+    "description": "",
+    "tables": "https:\/\/connection.keboola.com\/\/v2\/storage\/buckets\/in.c-bucket\/tables",
+    "created": "2016-11-07T01:57:53+0100",
+    "lastChangeDate": "2016-11-07T02:12:16+0100",
+    "isReadOnly": false,
+    "dataSizeBytes": 0,
+    "rowsCount": 0,
+    "isMaintenance": false,
+    "backend": "snowflake",
+    "metadata": []
+  }
+]
diff --git a/tests/data/backups/table-with-nullable-pk/configurations.json b/tests/data/backups/table-with-nullable-pk/configurations.json
@@ -0,0 +1 @@
+[]
diff --git a/tests/data/backups/table-with-nullable-pk/in/c-bucket/Account.csv.gz b/tests/data/backups/table-with-nullable-pk/in/c-bucket/Account.csv.gz
diff --git a/tests/data/backups/table-with-nullable-pk/in/c-bucket/Account2.csv.gz b/tests/data/backups/table-with-nullable-pk/in/c-bucket/Account2.csv.gz
diff --git a/tests/data/backups/table-with-nullable-pk/tables.json b/tests/data/backups/table-with-nullable-pk/tables.json
@@ -0,0 +1,104 @@
+[
+  {
+    "uri": "https://connection.eu-central-1.keboola.com/v2/storage/tables/in.c-native-tpes.firstTable",
+    "id": "in.c-bucket.firstTable",
+    "name": "firstTable",
+    "displayName": "firstTable",
+    "transactional": false,
+    "primaryKey": [
+      "Id"
+    ],
+    "indexType": null,
+    "indexKey": [],
+    "distributionType": null,
+    "distributionKey": [],
+    "syntheticPrimaryKeyEnabled": false,
+    "lastImportDate": null,
+    "rowsCount": null,
+    "dataSizeBytes": null,
+    "isAlias": false,
+    "isAliasable": true,
+    "isTyped": true,
+    "columns": [
+      "Id",
+      "Name"
+    ],
+    "bucket": {
+      "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/buckets\/in.c-bucket",
+      "id": "in.c-bucket",
+      "name": "c-bucket",
+      "stage": "in",
+      "description": "",
+      "tables": "https:\/\/connection.keboola.com\/\/v2\/storage\/buckets\/in.c-bucket\/tables",
+      "created": "2016-11-07T01:57:53+0100",
+      "lastChangeDate": "2016-11-07T02:12:16+0100",
+      "isReadOnly": false,
+      "dataSizeBytes": 6247936,
+      "rowsCount": 675727,
+      "isMaintenance": false,
+      "backend": "snowflake"
+    },
+    "metadata": [
+      {
+        "id": "1598834273",
+        "key": "KBC.dataTypesEnabled",
+        "value": "true",
+        "provider": "storage"
+      }
+    ],
+    "columnMetadata": {
+      "Id": [
+        {
+          "id": "1598834286",
+          "key": "KBC.datatype.type",
+          "value": "DATE",
+          "provider": "storage"
+        },
+        {
+          "id": "1598834286",
+          "key": "KBC.datatype.type",
+          "value": "VARCHAR",
+          "provider": "redshift"
+        },
+        {
+          "id": "1598834287",
+          "key": "KBC.datatype.nullable",
+          "value": "1",
+          "provider": "storage"
+        },
+        {
+          "id": "1598834288",
+          "key": "KBC.datatype.basetype",
+          "value": "DATE",
+          "provider": "storage"
+        }
+      ],
+      "Name": [
+        {
+          "id": "1598834282",
+          "key": "KBC.datatype.type",
+          "value": "TIMESTAMP_NTZ",
+          "provider": "storage"
+        },
+        {
+          "id": "1598834283",
+          "key": "KBC.datatype.nullable",
+          "value": "1",
+          "provider": "storage"
+        },
+        {
+          "id": "1598834284",
+          "key": "KBC.datatype.basetype",
+          "value": "TIMESTAMP",
+          "provider": "storage"
+        },
+        {
+          "id": "1598834285",
+          "key": "KBC.datatype.length",
+          "value": "9",
+          "provider": "storage"
+        }
+      ]
+    }
+  }
+]