Merge pull request #1493 from kuzudb/copy-rework

Add SERIAL as node table primary key

dataset/tinysnb-serial/copy.cypher

@@ -0,0 +1,8 @@
+COPY person FROM "dataset/tinysnb-serial/vPerson.csv" (HeaDER=true, deLim=',');
+COPY organisation FROM "dataset/tinysnb-serial/vOrganisation.csv";
+COPY movies FROM "dataset/tinysnb-serial/vMovies.csv";
+COPY knows FROM "dataset/tinysnb-serial/eKnows.csv";
+COPY studyAt FROM "dataset/tinysnb-serial/eStudyAt.csv" (HEADeR=true);
+COPY workAt FROM "dataset/tinysnb-serial/eWorkAt.csv"
+COPY meets FROM "dataset/tinysnb-serial/eMeets.csv"
+COPY marries FROM "dataset/tinysnb-serial/eMarries.csv"

dataset/tinysnb-serial/eKnows.csv

@@ -0,0 +1,14 @@
+0,1,2021-06-30,1986-10-21 21:08:31.521,10 years 5 months 13 hours 24 us,"[rnme,m8sihsdnf2990nfiwf]"
+0,2,2021-06-30,1946-08-25 19:07:22,20 years 30 days 48 hours,"[njnojppo9u0jkmf,fjiojioh9h9h89hph]"
+0,3,2021-06-30,2012-12-11 20:07:22,10 days,"[ioji232,jifhe8w99u43434]"
+1,0,2021-06-30,1946-08-25 19:07:22,10 years 5 months 13 hours 24 us,"[2huh9y89fsfw23,23nsihufhw723]"
+1,2,1950-05-14,1946-08-25 19:07:22,23 minutes,"[fwehu9h9832wewew,23u9h989sdfsss]"
+1,3,1950-05-14,2012-12-11 20:07:22,20 years 30 days 48 hours,"[fwh9y81232uisuiehuf,ewnuihxy8dyf232]"
+2,0,2021-06-30,2002-07-31 11:42:53.12342,30 hours 40 days,"[fnioh8323aeweae34d,osd89e2ejshuih12]"
+2,1,1950-05-14,2007-02-12 12:11:42.123,28 minutes 30 milliseconds,"[fwh983-sdjisdfji,ioh89y32r2huir]"
+2,3,2000-01-01,1998-10-02 13:09:22.423,300 milliseconds,"[psh989823oaaioe,nuiuah1nosndfisf]"
+3,0,2021-06-30,1936-11-02 11:02:01,480us,"[fwewe]"
+3,1,1950-05-14,1982-11-11 13:12:05.123,23 minutes,"[fewh9182912e3,h9y8y89soidfsf,nuhudf78w78efw,hioshe0f9023sdsd]"
+3,2,2000-01-01,1999-04-21 15:12:11.42,48 hours 52 milliseconds,"[23h9sdslnfowhu2932,shuhf98922323sf]"
+4,5,1905-12-12,2025-01-01 11:22:33.52,47 minutes 58 seconds,"[ahu2333333333333,12weeeeeeeeeeeeeeeeee]"
+4,6,1905-12-12,2020-03-01 12:11:41.6552,47 minutes 58 seconds,"[peweeeeeeeeeeeeeeeee,kowje9w0eweeeeeeeee]"

dataset/tinysnb-serial/eMarries.csv

@@ -0,0 +1,3 @@
+0,1,"[toronto]","[4,5]",
+2,3,,"[2,5]","long long long string"
+4,5,"[]","[3,9]","short str"

dataset/tinysnb-serial/eMeets.csv

@@ -0,0 +1,7 @@
+0,1,"[7.82,3.54]",5
+1,3,"[2.87,4.23]",2
+2,4,"[3.65,8.44]",3
+4,2,"[2.11,3.1]",7
+5,2,"[2.2,9.0]",9
+6,2,"[3,5.2]",11
+7,1,"[3.5,1.1]",13

dataset/tinysnb-serial/eStudyAt.csv

@@ -0,0 +1,4 @@
+from,to,YEAR,Places,length
+0,0,2021,"[wwAewsdndweusd,wek]",5
+1,0,2020,"[anew,jsdnwusklklklwewsd]",55
+5,0,2020,"[awndsnjwejwen,isuhuwennjnuhuhuwewe]",22

dataset/tinysnb-serial/eWorkAt.csv

@@ -0,0 +1,3 @@
+2,1,2015,"[3.8,2.5]",8.2
+3,2,2010,"[2.1,4.4]",7.6
+4,2,2015,"[9.2,3.1]",9.2

dataset/tinysnb-serial/schema.cypher

@@ -0,0 +1,8 @@
+create node table person (ID SERIAL, fName StRING, gender INT64, isStudent BoOLEAN, isWorker BOOLEAN, age INT64, eyeSight DOUBLE, birthdate DATE, registerTime TIMESTAMP, lastJobDuration interval, workedHours INT64[], usedNames STRING[], courseScoresPerTerm INT64[][], grades INT64[4], height float, PRIMARY KEY (ID));
+create node table organisation (ID SERIAL, name STRING, orgCode INT64, mark DOUBLE, score INT64, history STRING, licenseValidInterval INTERVAL, rating DOUBLE, PRIMARY KEY (ID));
+create node table movies (ID SERIAL, name STRING, length INT32, note STRING, PRIMARY KEY (ID));
+create rel table knows (FROM person TO person, date DATE, meetTime TIMESTAMP, validInterval INTERVAL, comments STRING[], MANY_MANY);
+create rel table studyAt (FROM person TO organisation, year INT64, places STRING[], length INT16,MANY_ONE);
+create rel table workAt (FROM person TO organisation, year INT64, grading DOUBLE[2], rating float, MANY_ONE);
+create rel table meets (FROM person TO person, location FLOAT[2], times INT, MANY_ONE);
+create rel table marries (FROM person TO person, usedAddress STRING[], address INT16[2], note STRING, ONE_ONE);

dataset/tinysnb-serial/vMovies.csv

@@ -0,0 +1,3 @@
+Sóló cón tu párejâ,126, this is a very very good movie
+The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie,2544, the movie is very very good
+Roma,298,the movie is very interesting and funny

dataset/tinysnb-serial/vOrganisation.csv

@@ -0,0 +1,3 @@
+ABFsUni,325,3.7,-2,10 years 5 months 13 hours 24 us,3 years 5 days,1
+CsWork,934,4.1,-100,2 years 4 days 10 hours,26 years 52 days 48 hours,0.78
+DEsWork,824,4.1,7,2 years 4 hours 22 us 34 minutes,82 hours 100 milliseconds,0.52

dataset/tinysnb-serial/vPerson.csv

@@ -0,0 +1,9 @@
+fname,Gender,ISStudent,isWorker,age,eyeSight,birthdate,registerTime,lastJobDuration,workedHours,usedNames,courseScoresPerTerm,grades,height
+Alice,1,true,false,35,5.0,1900-01-01,2011-08-20 11:25:30Z+00:00,3 years 2 days 13 hours 2 minutes,"[10,5]","[Aida]","[[10,8],[6,7,8]]","[96,54,86,92]",1.731
+Bob,2,true,false,30,5.1,1900-01-01,2008-11-03 13:25:30.000526-02:00,10 years 5 months 13 hours 24 us,"[12,8]","[Bobby]","[[8,9],[9,10]]","[98,42,93,88]",0.99
+Carol,1,false,true,45,5.0,1940-06-22,1911-08-20 02:32:21,48 hours 24 minutes 11 seconds,"[4,5]","[Carmen,Fred]","[[8,10]]","[91,75,21,95]",1.00
+Dan,2,false,true,20,4.8,1950-7-23,2031-11-30 12:25:30Z,10 years 5 months 13 hours 24 us,"[1,9]","[Wolfeschlegelstein,Daniel]","[[7,4],[8,8],[9]]","[76,88,99,89]",1.30
+Elizabeth,1,false,true,20,4.7,1980-10-26,1976-12-23 11:21:42,48 hours 24 minutes 11 seconds,"[2]","[Ein]","[[6],[7],[8]]","[96,59,65,88]",1.463
+Farooq,2,true,false,25,4.5,1980-10-26,1972-07-31 13:22:30.678559,18 minutes 24 milliseconds,"[3,4,5,6,7]","[Fesdwe]","[[8]]","[80,78,34,83]",1.51
+Greg,2,false,false,40,4.9,1980-10-26,1976-12-23 11:21:42Z+06:40,10 years 5 months 13 hours 24 us,"[1]","[Grad]","[[10]]","[43,83,67,43]",1.6
+Hubert Blaine Wolfeschlegelsteinhausenbergerdorff,2,false,true,83,4.9,1990-11-27,2023-02-21 13:25:30,3 years 2 days 13 hours 2 minutes,"[10,11,12,3,4,5,6,7]","[Ad,De,Hi,Kye,Orlan]","[[7],[10],[6,7]]","[77,64,100,54]",1.323

src/binder/bind/bind_ddl.cpp

@@ -22,25 +22,37 @@ using namespace kuzu::catalog;
 namespace kuzu {
 namespace binder {
 
-std::unique_ptr<BoundStatement> Binder::bindCreateNodeClause(const Statement& statement) {
-    auto& createNodeClause = (parser::CreateNodeClause&)statement;
-    auto tableName = createNodeClause.getTableName();
+std::unique_ptr<BoundStatement> Binder::bindCreateNodeTableClause(
+    const parser::Statement& statement) {
+    auto& createNodeTableClause = (parser::CreateNodeTableClause&)statement;
+    auto tableName = createNodeTableClause.getTableName();
     if (catalog.getReadOnlyVersion()->containTable(tableName)) {
         throw BinderException("Node " + tableName + " already exists.");
     }
-    auto boundProperties = bindProperties(createNodeClause.getPropertyNameDataTypes());
+    auto boundProperties = bindProperties(createNodeTableClause.getPropertyNameDataTypes());
     auto primaryKeyIdx = bindPrimaryKey(
-        createNodeClause.getPKColName(), createNodeClause.getPropertyNameDataTypes());
+        createNodeTableClause.getPKColName(), createNodeTableClause.getPropertyNameDataTypes());
+    for (auto i = 0u; i < boundProperties.size(); ++i) {
+        if (boundProperties[i].dataType.typeID == SERIAL && primaryKeyIdx != i) {
+            throw BinderException("Serial property in node table must be the primary key.");
+        }
+    }
     return make_unique<BoundCreateNodeClause>(tableName, std::move(boundProperties), primaryKeyIdx);
 }
 
-std::unique_ptr<BoundStatement> Binder::bindCreateRelClause(const Statement& statement) {
+std::unique_ptr<BoundStatement> Binder::bindCreateRelTableClause(
+    const parser::Statement& statement) {
     auto& createRelClause = (CreateRelClause&)statement;
     auto tableName = createRelClause.getTableName();
     if (catalog.getReadOnlyVersion()->containTable(tableName)) {
         throw BinderException("Rel " + tableName + " already exists.");
     }
     auto boundProperties = bindProperties(createRelClause.getPropertyNameDataTypes());
+    for (auto& boundProperty : boundProperties) {
+        if (boundProperty.dataType.typeID == SERIAL) {
+            throw BinderException("Serial property is not supported in rel table.");
+        }
+    }
     auto relMultiplicity = getRelMultiplicityFromString(createRelClause.getRelMultiplicity());
     return make_unique<BoundCreateRelClause>(tableName, std::move(boundProperties), relMultiplicity,
         bindNodeTableID(createRelClause.getSrcTableName()),
@@ -81,6 +93,9 @@ std::unique_ptr<BoundStatement> Binder::bindAddPropertyClause(const parser::Stat
     if (catalogContent->getTableSchema(tableID)->containProperty(addProperty.getPropertyName())) {
         throw BinderException("Property: " + addProperty.getPropertyName() + " already exists.");
     }
+    if (dataType.typeID == SERIAL) {
+        throw BinderException("Serial property in node table must be the primary key.");
+    }
     auto defaultVal = ExpressionBinder::implicitCastIfNecessary(
         expressionBinder.bindExpression(*addProperty.getDefaultValue()), dataType);
     return make_unique<BoundAddProperty>(
@@ -157,10 +172,11 @@ uint32_t Binder::bindPrimaryKey(const std::string& pkColName,
     }
     auto primaryKey = propertyNameDataTypes[primaryKeyIdx];
     StringUtils::toUpper(primaryKey.second);
-    // We only support INT64, and STRING column as the primary key.
+    // We only support INT64, STRING and SERIAL column as the primary key.
     switch (Types::dataTypeFromString(primaryKey.second).typeID) {
     case common::INT64:
     case common::STRING:
+    case common::SERIAL:
         break;
     default:
         throw BinderException(

src/binder/binder.cpp

@@ -12,11 +12,11 @@ namespace binder {
 
 std::unique_ptr<BoundStatement> Binder::bind(const Statement& statement) {
     switch (statement.getStatementType()) {
-    case StatementType::CREATE_NODE_CLAUSE: {
-        return bindCreateNodeClause(statement);
+    case StatementType::CREATE_NODE_TABLE_CLAUSE: {
+        return bindCreateNodeTableClause(statement);
     }
-    case StatementType::CREATE_REL_CLAUSE: {
-        return bindCreateRelClause(statement);
+    case StatementType::CREATE_REL_TABLE_CLAUSE: {
+        return bindCreateRelTableClause(statement);
     }
     case StatementType::COPY: {
         return bindCopyClause(statement);

src/common/types/types.cpp

@@ -332,6 +332,8 @@ DataTypeID Types::dataTypeIDFromString(const std::string& dataTypeIDString) {
         return TIMESTAMP;
     } else if ("INTERVAL" == dataTypeIDString) {
         return INTERVAL;
+    } else if ("SERIAL" == dataTypeIDString) {
+        return SERIAL;
     } else {
         throw InternalException("Cannot parse dataTypeID: " + dataTypeIDString);
     }
@@ -415,6 +417,8 @@ std::string Types::dataTypeToString(DataTypeID dataTypeID) {
         return "FIXED_LIST";
     case STRUCT:
         return "STRUCT";
+    case SERIAL:
+        return "SERIAL";
     default:
         throw InternalException(
             "Unsupported DataType: " + Types::dataTypeToString(dataTypeID) + ".");
@@ -447,6 +451,7 @@ uint32_t Types::getDataTypeSize(DataTypeID dataTypeID) {
         return sizeof(internalID_t);
     case BOOL:
         return sizeof(uint8_t);
+    case SERIAL:
     case INT64:
         return sizeof(int64_t);
     case INT32:
@@ -490,6 +495,7 @@ uint32_t Types::getDataTypeSize(const DataType& dataType) {
     }
     case INTERNAL_ID:
     case BOOL:
+    case SERIAL:
     case INT64:
     case INT32:
     case INT16:

src/common/types/value.cpp

@@ -324,18 +324,6 @@ Value::Value() : dataType{ANY}, isNull_{true} {}
 
 Value::Value(DataType dataType) : dataType{std::move(dataType)}, isNull_{true} {}
 
-void Value::validateType(DataTypeID typeID) const {
-    validateType(DataType(typeID));
-}
-
-void Value::validateType(const DataType& type) const {
-    if (type != dataType) {
-        throw RuntimeException(
-            StringUtils::string_format("Cannot get {} value from the {} result value.",
-                Types::dataTypeToString(type), Types::dataTypeToString(dataType)));
-    }
-}
-
 std::vector<std::unique_ptr<Value>> Value::convertKUVarListToVector(ku_list_t& list) const {
     std::vector<std::unique_ptr<Value>> listResultValue;
     auto numBytesPerElement = Types::getDataTypeSize(*dataType.getChildType());

src/include/binder/binder.h

@@ -39,8 +39,8 @@ class Binder {
         const std::string& name, const common::DataType& dataType);
 
     /*** bind DDL ***/
-    std::unique_ptr<BoundStatement> bindCreateNodeClause(const parser::Statement& statement);
-    std::unique_ptr<BoundStatement> bindCreateRelClause(const parser::Statement& statement);
+    std::unique_ptr<BoundStatement> bindCreateNodeTableClause(const parser::Statement& statement);
+    std::unique_ptr<BoundStatement> bindCreateRelTableClause(const parser::Statement& statement);
     std::unique_ptr<BoundStatement> bindDropTableClause(const parser::Statement& statement);
     std::unique_ptr<BoundStatement> bindRenameTableClause(const parser::Statement& statement);
     std::unique_ptr<BoundStatement> bindAddPropertyClause(const parser::Statement& statement);

src/include/binder/ddl/bound_create_node_clause.h

@@ -9,7 +9,7 @@ class BoundCreateNodeClause : public BoundCreateTable {
 public:
     explicit BoundCreateNodeClause(
         std::string tableName, std::vector<catalog::Property> properties, uint32_t primaryKeyIdx)
-        : BoundCreateTable{common::StatementType::CREATE_NODE_CLAUSE, std::move(tableName),
+        : BoundCreateTable{common::StatementType::CREATE_NODE_TABLE_CLAUSE, std::move(tableName),
               std::move(properties)},
           primaryKeyIdx{primaryKeyIdx} {}
 

src/include/binder/ddl/bound_create_rel_clause.h

@@ -10,7 +10,7 @@ class BoundCreateRelClause : public BoundCreateTable {
     BoundCreateRelClause(std::string tableName, std::vector<catalog::Property> properties,
         catalog::RelMultiplicity relMultiplicity, common::table_id_t srcTableID,
         common::table_id_t dstTableID)
-        : BoundCreateTable{common::StatementType::CREATE_REL_CLAUSE, std::move(tableName),
+        : BoundCreateTable{common::StatementType::CREATE_REL_TABLE_CLAUSE, std::move(tableName),
               std::move(properties)},
           relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID} {}
 

src/include/catalog/catalog_structs.h

@@ -17,6 +17,9 @@ std::string getRelMultiplicityAsString(RelMultiplicity relMultiplicity);
 
 struct Property {
 public:
+    static constexpr std::string_view REL_FROM_PROPERTY_NAME = "_FROM_";
+    static constexpr std::string_view REL_TO_PROPERTY_NAME = "_TO_";
+
     // This constructor is needed for ser/deser functions
     Property() : Property{"", common::DataType{}} {};
     Property(std::string name, common::DataType dataType)

src/include/common/statement_type.h

@@ -7,8 +7,8 @@ namespace common {
 
 enum class StatementType : uint8_t {
     QUERY = 0,
-    CREATE_NODE_CLAUSE = 1,
-    CREATE_REL_CLAUSE = 2,
+    CREATE_NODE_TABLE_CLAUSE = 1,
+    CREATE_REL_TABLE_CLAUSE = 2,
     COPY = 3,
     DROP_TABLE = 4,
     RENAME_TABLE = 5,
@@ -20,8 +20,8 @@ enum class StatementType : uint8_t {
 class StatementTypeUtils {
 public:
     static bool isDDL(StatementType statementType) {
-        return statementType == StatementType::CREATE_NODE_CLAUSE ||
-               statementType == StatementType::CREATE_REL_CLAUSE ||
+        return statementType == StatementType::CREATE_NODE_TABLE_CLAUSE ||
+               statementType == StatementType::CREATE_REL_TABLE_CLAUSE ||
                statementType == StatementType::DROP_TABLE ||
                statementType == StatementType::DROP_PROPERTY;
     }

src/include/common/types/types.h

@@ -53,6 +53,9 @@ KUZU_API enum DataTypeID : uint8_t {
     ANY = 0,
     NODE = 10,
     REL = 11,
+    // SERIAL is a special data type that is used to represent a sequence of INT64 values that are
+    // incremented by 1 starting from 0.
+    SERIAL = 12,
 
     // physical types
 

src/include/common/types/value.h

@@ -184,9 +184,6 @@ class Value {
     Value();
     explicit Value(DataType dataType);
 
-    void validateType(DataTypeID typeID) const;
-    void validateType(const DataType& type) const;
-
     template<typename T>
     static inline void putValuesIntoVector(std::vector<std::unique_ptr<Value>>& fixedListResultVal,
         const uint8_t* fixedList, uint64_t numBytesPerElement) {

src/include/parser/ddl/create_node_clause.h

@@ -5,12 +5,12 @@
 namespace kuzu {
 namespace parser {
 
-class CreateNodeClause : public CreateTable {
+class CreateNodeTableClause : public CreateTable {
 public:
-    explicit CreateNodeClause(std::string tableName,
+    explicit CreateNodeTableClause(std::string tableName,
         std::vector<std::pair<std::string, std::string>> propertyNameDataTypes,
         std::string pkColName)
-        : CreateTable{common::StatementType::CREATE_NODE_CLAUSE, std::move(tableName),
+        : CreateTable{common::StatementType::CREATE_NODE_TABLE_CLAUSE, std::move(tableName),
               std::move(propertyNameDataTypes)},
           pKColName{std::move(pkColName)} {}
 

src/include/parser/ddl/create_rel_clause.h

@@ -10,7 +10,7 @@ class CreateRelClause : public CreateTable {
     CreateRelClause(std::string tableName,
         std::vector<std::pair<std::string, std::string>> propertyNameDataTypes,
         std::string relMultiplicity, std::string srcTableName, std::string dstTableName)
-        : CreateTable{common::StatementType::CREATE_REL_CLAUSE, std::move(tableName),
+        : CreateTable{common::StatementType::CREATE_REL_TABLE_CLAUSE, std::move(tableName),
               std::move(propertyNameDataTypes)},
           relMultiplicity{std::move(relMultiplicity)}, srcTableName{std::move(srcTableName)},
           dstTableName{std::move(dstTableName)} {}

src/include/storage/copier/node_copier.h

@@ -99,7 +99,11 @@ class NodeCopier {
 
     void execute(processor::ExecutionContext* executionContext);
 
-    inline virtual void finalize() { pkIndex->flush(); }
+    inline virtual void finalize() {
+        if (pkIndex) {
+            pkIndex->flush();
+        }
+    }
 
     virtual std::unique_ptr<NodeCopier> clone() const {
         return std::make_unique<NodeCopier>(sharedState, pkIndex, copyDesc, columns, pkColumnID);
@@ -191,12 +195,6 @@ class NPYNodeCopier : public NodeCopier {
             this->sharedState, this->pkIndex, this->copyDesc, this->columns, this->pkColumnID);
     }
 
-    inline void finalize() override {
-        if (this->pkColumnID != common::INVALID_COLUMN_ID) {
-            this->pkIndex->flush();
-        }
-    }
-
 protected:
     void executeInternal(std::unique_ptr<NodeCopyMorsel> morsel) override;
 

src/include/storage/copier/rel_copy_executor.h

@@ -37,7 +37,7 @@ class RelCopyExecutor : public TableCopyExecutor {
 
     void initListsMetadata();
 
-    void initializePkIndexes(common::table_id_t nodeTableID, BufferManager& bufferManager);
+    void initializePkIndexes(common::table_id_t nodeTableID);
 
     void executePopulateTask(PopulateTaskType populateTaskType);