Merge pull request #2208 from kuzudb/utf8-check

Throw copy exception on invalid utf8 string
kuzudb · Oct 13, 2023 · 3280465 · 3280465
2 parents 964925f + 11f3a52
commit 3280465
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 9 deletions.
diff --git a/dataset/invalid-utf8/invalid-utf8.csv → ...fault-tests/invalid-utf8/invalid-utf8.csv b/dataset/invalid-utf8/invalid-utf8.csv → ...fault-tests/invalid-utf8/invalid-utf8.csv
diff --git a/dataset/copy-fault-tests/invalid-utf8/schema.cypher b/dataset/copy-fault-tests/invalid-utf8/schema.cypher
@@ -0,0 +1 @@
+create node table person (ID STRING, PRIMARY KEY (ID));
diff --git a/src/processor/operator/persistent/reader/csv/driver.cpp b/src/processor/operator/persistent/reader/csv/driver.cpp
@@ -11,6 +11,7 @@
 #include "processor/operator/persistent/reader/csv/parallel_csv_reader.h"
 #include "processor/operator/persistent/reader/csv/serial_csv_reader.h"
 #include "storage/store/table_copy_utils.h"
+#include "utf8proc_wrapper.h"
 
 using namespace kuzu::common;
 
@@ -461,6 +462,9 @@ void copyStringToVector(ValueVector* vector, uint64_t rowToAdd, std::string_view
             vector, rowToAdd, reinterpret_cast<char*>(blobBuffer.get()), blobLen);
     } break;
     case LogicalTypeID::STRING: {
+        if (!utf8proc::Utf8Proc::isValid(strVal.data(), strVal.length())) {
+            throw common::CopyException{"Invalid UTF8-encoded string."};
+        }
         StringVector::addString(vector, rowToAdd, strVal.data(), strVal.length());
     } break;
     case LogicalTypeID::DATE: {

diff --git a/test/test_files/exceptions/copy/invalid_utf8.test b/test/test_files/exceptions/copy/invalid_utf8.test
@@ -0,0 +1,9 @@
+-GROUP CopyInvalidUTF8
+-DATASET CSV copy-fault-tests/invalid-utf8
+
+--
+
+-CASE InvalidUTF8
+-STATEMENT COPY person FROM "${KUZU_ROOT_DIRECTORY}/dataset/copy-fault-tests/invalid-utf8/invalid-utf8.csv"
+---- error
+Copy exception: Invalid UTF8-encoded string.
diff --git a/test/test_files/exceptions/runtime/invalid_utf8.test b/test/test_files/exceptions/runtime/invalid_utf8.test