Add fixed-list,union,map functions

src/common/type_utils.cpp

@@ -2,7 +2,7 @@
 
 #include "common/exception/conversion.h"
 #include "common/exception/runtime.h"
-#include "common/string_utils.h"
+#include "common/types/blob.h"
 #include "common/vector/value_vector.h"
 
 namespace kuzu {
@@ -56,6 +56,11 @@ std::string TypeUtils::castValueToString(
     }
 }
 
+template<>
+std::string TypeUtils::toString(const int128_t& val, void* /*valueVector*/) {
+    return Int128_t::ToString(val);
+}
+
 template<>
 std::string TypeUtils::toString(const bool& val, void* /*valueVector*/) {
     return val ? "True" : "False";
@@ -86,6 +91,11 @@ std::string TypeUtils::toString(const ku_string_t& val, void* /*valueVector*/) {
     return val.getAsString();
 }
 
+template<>
+std::string TypeUtils::toString(const blob_t& val, void* /*valueVector*/) {
+    return Blob::toString(val.value.getData(), val.value.len);
+}
+
 template<>
 std::string TypeUtils::toString(const list_entry_t& val, void* valueVector) {
     auto listVector = (ValueVector*)valueVector;
@@ -168,10 +178,15 @@ std::string TypeUtils::toString(const struct_entry_t& val, void* valVector) {
     return result;
 }
 
-std::string TypeUtils::prefixConversionExceptionMessage(
-    const char* data, LogicalTypeID dataTypeID) {
-    return "Cannot convert string " + std::string(data) + " to " +
-           LogicalTypeUtils::dataTypeToString(dataTypeID) + ".";
+template<>
+std::string TypeUtils::toString(const union_entry_t& val, void* valVector) {
+    auto structVector = (ValueVector*)valVector;
+    auto unionFieldIdx =
+        UnionVector::getTagVector(structVector)->getValue<union_field_idx_t>(val.entry.pos);
+    auto unionFieldVector = UnionVector::getValVector(structVector, unionFieldIdx);
+    return castValueToString(unionFieldVector->dataType,
+        unionFieldVector->getData() + unionFieldVector->getNumBytesPerValue() * val.entry.pos,
+        unionFieldVector);
 }
 
 } // namespace common

src/common/types/types.cpp

@@ -745,8 +745,6 @@ std::vector<LogicalTypeID> LogicalTypeUtils::getIntegerLogicalTypeIDs() {
 }
 
 std::vector<LogicalType> LogicalTypeUtils::getAllValidLogicTypes() {
-    // TODO(Ziyi): Add FIX_LIST,MAP type to allValidTypeID when we support functions on
-    // FIXED_LIST,MAP.
     return std::vector<LogicalType>{LogicalType{LogicalTypeID::INTERNAL_ID},
         LogicalType{LogicalTypeID::BOOL}, LogicalType{LogicalTypeID::INT64},
         LogicalType{LogicalTypeID::INT32}, LogicalType{LogicalTypeID::INT16},
@@ -756,9 +754,11 @@ std::vector<LogicalType> LogicalTypeUtils::getAllValidLogicTypes() {
         LogicalType{LogicalTypeID::DOUBLE}, LogicalType{LogicalTypeID::STRING},
         LogicalType{LogicalTypeID::BLOB}, LogicalType{LogicalTypeID::DATE},
         LogicalType{LogicalTypeID::TIMESTAMP}, LogicalType{LogicalTypeID::INTERVAL},
-        LogicalType{LogicalTypeID::VAR_LIST}, LogicalType{LogicalTypeID::FLOAT},
+        LogicalType{LogicalTypeID::VAR_LIST}, LogicalType{LogicalTypeID::FIXED_LIST},
+        LogicalType{LogicalTypeID::MAP}, LogicalType{LogicalTypeID::FLOAT},
         LogicalType{LogicalTypeID::SERIAL}, LogicalType{LogicalTypeID::NODE},
-        LogicalType{LogicalTypeID::REL}, LogicalType{LogicalTypeID::STRUCT}};
+        LogicalType{LogicalTypeID::REL}, LogicalType{LogicalTypeID::STRUCT},
+        LogicalType{LogicalTypeID::UNION}};
 }
 
 std::vector<std::string> LogicalTypeUtils::parseStructFields(const std::string& structTypeStr) {

src/function/vector_cast_functions.cpp

@@ -166,8 +166,48 @@ vector_function_definitions CastToIntervalVectorFunction::getDefinitions() {
     return result;
 }
 
+void castFixedListToString(
+    ValueVector& param, uint64_t pos, ValueVector& resultVector, uint64_t resultPos) {
+    resultVector.setNull(resultPos, param.isNull(pos));
+    if (param.isNull(pos)) {
+        return;
+    }
+    std::string result = "[";
+    auto numValuesPerList = FixedListType::getNumElementsInList(&param.dataType);
+    auto childType = FixedListType::getChildType(&param.dataType);
+    auto values = param.getData() + pos * param.getNumBytesPerValue();
+    for (auto i = 0u; i < numValuesPerList - 1; ++i) {
+        // Note: FixedList can only store numeric types and doesn't allow nulls.
+        result += TypeUtils::castValueToString(*childType, values, nullptr /* vector */);
+        result += ",";
+        values += PhysicalTypeUtils::getFixedTypeSize(childType->getPhysicalType());
+    }
+    result += TypeUtils::castValueToString(*childType, values, nullptr /* vector */);
+    result += "]";
+    resultVector.setValue(resultPos, result);
+}
+
+void fixedListCastExecFunction(
+    const std::vector<std::shared_ptr<ValueVector>>& params, common::ValueVector& result) {
+    assert(params.size() == 1);
+    auto param = params[0];
+    if (param->state->isFlat()) {
+        castFixedListToString(*param, param->state->selVector->selectedPositions[0], result,
+            result.state->selVector->selectedPositions[0]);
+    } else if (param->state->selVector->isUnfiltered()) {
+        for (auto i = 0u; i < param->state->selVector->selectedSize; i++) {
+            castFixedListToString(*param, i, result, i);
+        }
+    } else {
+        for (auto i = 0u; i < param->state->selVector->selectedSize; i++) {
+            castFixedListToString(*param, param->state->selVector->selectedPositions[i], result,
+                result.state->selVector->selectedPositions[i]);
+        }
+    }
+}
+
 void CastToStringVectorFunction::getUnaryCastToStringExecFunction(
-    common::LogicalTypeID typeID, scalar_exec_func& func) {
+    LogicalTypeID typeID, scalar_exec_func& func) {
     switch (typeID) {
     case common::LogicalTypeID::BOOL: {
         func = UnaryCastExecFunction<bool, ku_string_t, CastToString>;
@@ -196,7 +236,7 @@ void CastToStringVectorFunction::getUnaryCastToStringExecFunction(
     } break;
     case common::LogicalTypeID::INT128: {
         func = UnaryCastExecFunction<int128_t, ku_string_t, CastToString>;
-    }
+    } break;
     case common::LogicalTypeID::UINT8: {
         func = UnaryCastExecFunction<uint8_t, ku_string_t, CastToString>;
     } break;
@@ -218,20 +258,28 @@ void CastToStringVectorFunction::getUnaryCastToStringExecFunction(
     case common::LogicalTypeID::INTERNAL_ID: {
         func = UnaryCastExecFunction<internalID_t, ku_string_t, CastToString>;
     } break;
-    case common::LogicalTypeID::BLOB:
+    case common::LogicalTypeID::BLOB: {
+        func = UnaryCastExecFunction<blob_t, ku_string_t, CastToString>;
+    } break;
     case common::LogicalTypeID::STRING: {
         func = UnaryCastExecFunction<ku_string_t, ku_string_t, CastToString>;
     } break;
     case common::LogicalTypeID::VAR_LIST: {
         func = UnaryCastExecFunction<list_entry_t, ku_string_t, CastToString>;
     } break;
+    case common::LogicalTypeID::FIXED_LIST: {
+        func = fixedListCastExecFunction;
+    } break;
     case common::LogicalTypeID::MAP: {
         func = UnaryCastExecFunction<map_entry_t, ku_string_t, CastToString>;
     } break;
     case common::LogicalTypeID::NODE:
     case common::LogicalTypeID::REL:
     case common::LogicalTypeID::STRUCT: {
         func = UnaryCastExecFunction<struct_entry_t, ku_string_t, CastToString>;
+    } break;
+    case common::LogicalTypeID::UNION: {
+        func = UnaryCastExecFunction<union_entry_t, ku_string_t, CastToString>;
     } break;
         // LCOV_EXCL_START
     default:

src/include/common/type_utils.h

@@ -12,6 +12,9 @@
 
 namespace kuzu {
 namespace common {
+
+struct blob_t;
+
 class TypeUtils {
 
 public:
@@ -37,20 +40,14 @@ class TypeUtils {
         memcpy(&pageOffset, ((uint8_t*)&overflowPtr) + 4, 2);
     }
 
-    static std::string prefixConversionExceptionMessage(const char* data, LogicalTypeID dataTypeID);
-
-private:
     static std::string castValueToString(
         const LogicalType& dataType, const uint8_t* value, void* vector);
 };
 
-template<>
-inline std::string TypeUtils::toString(const int128_t& val, void* /*valueVector*/) {
-    return Int128_t::ToString(val);
-}
-
 // Forward declaration of template specializations.
 template<>
+std::string TypeUtils::toString(const int128_t& val, void* valueVector);
+template<>
 std::string TypeUtils::toString(const bool& val, void* valueVector);
 template<>
 std::string TypeUtils::toString(const internalID_t& val, void* valueVector);
@@ -63,11 +60,15 @@ std::string TypeUtils::toString(const interval_t& val, void* valueVector);
 template<>
 std::string TypeUtils::toString(const ku_string_t& val, void* valueVector);
 template<>
+std::string TypeUtils::toString(const blob_t& val, void* valueVector);
+template<>
 std::string TypeUtils::toString(const list_entry_t& val, void* valueVector);
 template<>
 std::string TypeUtils::toString(const map_entry_t& val, void* valueVector);
 template<>
 std::string TypeUtils::toString(const struct_entry_t& val, void* valueVector);
+template<>
+std::string TypeUtils::toString(const union_entry_t& val, void* valueVector);
 
 } // namespace common
 } // namespace kuzu

src/include/common/types/types.h

@@ -35,10 +35,9 @@ using vector_idx_t = uint32_t;
 constexpr vector_idx_t INVALID_VECTOR_IDX = UINT32_MAX;
 using block_idx_t = uint64_t;
 constexpr block_idx_t INVALID_BLOCK_IDX = UINT64_MAX;
-using field_idx_t = uint64_t;
-using struct_field_idx_t = uint64_t;
-using union_field_idx_t = uint64_t;
-constexpr struct_field_idx_t INVALID_STRUCT_FIELD_IDX = UINT64_MAX;
+using struct_field_idx_t = uint8_t;
+using union_field_idx_t = struct_field_idx_t;
+constexpr struct_field_idx_t INVALID_STRUCT_FIELD_IDX = UINT8_MAX;
 using row_idx_t = uint64_t;
 constexpr row_idx_t INVALID_ROW_IDX = UINT64_MAX;
 constexpr uint32_t UNDEFINED_CAST_COST = UINT32_MAX;
@@ -69,7 +68,7 @@ struct map_entry_t {
 };
 
 struct union_entry_t {
-    int64_t pos;
+    struct_entry_t entry;
 };
 
 enum class KUZU_API LogicalTypeID : uint8_t {
@@ -396,7 +395,7 @@ struct MapType {
 struct UnionType {
     static constexpr union_field_idx_t TAG_FIELD_IDX = 0;
 
-    static constexpr LogicalTypeID TAG_FIELD_TYPE = LogicalTypeID::INT64;
+    static constexpr LogicalTypeID TAG_FIELD_TYPE = LogicalTypeID::INT8;
 
     static constexpr char TAG_FIELD_NAME[] = "tag";
 

src/include/common/vector/value_vector.h

@@ -212,6 +212,11 @@ class UnionVector {
         return StructVector::getFieldVector(vector, UnionType::TAG_FIELD_IDX).get();
     }
 
+    static inline ValueVector* getValVector(const ValueVector* vector, union_field_idx_t fieldIdx) {
+        assert(vector->dataType.getLogicalTypeID() == LogicalTypeID::UNION);
+        return StructVector::getFieldVector(vector, UnionType::getInternalFieldIdx(fieldIdx)).get();
+    }
+
     static inline void referenceVector(ValueVector* vector, union_field_idx_t fieldIdx,
         std::shared_ptr<ValueVector> vectorToReference) {
         StructVector::referenceVector(

src/include/function/cast/functions/cast_functions.h

@@ -14,15 +14,10 @@ namespace kuzu {
 namespace function {
 
 struct CastToString {
-    template<typename T>
-    static inline std::string castToString(T& input, const common::ValueVector& inputVector) {
-        return common::TypeUtils::toString(input, (void*)&inputVector);
-    }
-
     template<typename T>
     static inline void operation(T& input, common::ku_string_t& result,
         common::ValueVector& inputVector, common::ValueVector& resultVector) {
-        std::string resultStr = castToString(input, inputVector);
+        std::string resultStr = common::TypeUtils::toString(input, (void*)&inputVector);
         if (resultStr.length() > common::ku_string_t::SHORT_STR_LENGTH) {
             result.overflowPtr = reinterpret_cast<uint64_t>(
                 common::StringVector::getInMemOverflowBuffer(&resultVector)

src/include/function/union/functions/union_tag.h

@@ -10,7 +10,7 @@ struct UnionTag {
         common::ValueVector& unionVector, common::ValueVector& tagVector) {
         auto tagIdxVector = common::UnionVector::getTagVector(&unionVector);
         auto tagIdx = tagIdxVector->getValue<common::union_field_idx_t>(
-            tagIdxVector->state->selVector->selectedPositions[unionValue.pos]);
+            tagIdxVector->state->selVector->selectedPositions[unionValue.entry.pos]);
         auto tagName = common::UnionType::getFieldName(&unionVector.dataType, tagIdx);
         if (tagName.length() > common::ku_string_t::SHORT_STR_LENGTH) {
             tag.overflowPtr =

test/test_files/copy/copy_to_csv.test

@@ -5,26 +5,26 @@
 
 -CASE TinySnbCopyToCSV
 
--STATEMENT COPY (MATCH (p:person) RETURN [id(p)], p.ID, p.fName, p.gender, p.isStudent, p.age, p.eyeSight, p.birthdate, p.registerTime, p.lastJobDuration, p.workedHours, p.usedNames, p.courseScoresPerTerm, p.height) TO "${DATABASE_PATH}/tinysnb.csv"
+-STATEMENT COPY (MATCH (p:person) RETURN [id(p)], p.*) TO "${DATABASE_PATH}/tinysnb.csv"
 ---- ok
 -STATEMENT load from "${DATABASE_PATH}/tinysnb.csv" return *
 ---- 8
-[0:0]|0|Alice|1|True|35|5.000000|1900-01-01|2011-08-20 11:25:30|3 years 2 days 13:02:00|[10,5]|[Aida]|[[10,8],[6,7,8]]|1.731000
-[0:1]|2|Bob|2|True|30|5.100000|1900-01-01|2008-11-03 15:25:30.000526|10 years 5 months 13:00:00.000024|[12,8]|[Bobby]|[[8,9],[9,10]]|0.990000
-[0:2]|3|Carol|1|False|45|5.000000|1940-06-22|1911-08-20 02:32:21|48:24:11|[4,5]|[Carmen,Fred]|[[8,10]]|1.000000
-[0:3]|5|Dan|2|False|20|4.800000|1950-07-23|2031-11-30 12:25:30|10 years 5 months 13:00:00.000024|[1,9]|[Wolfeschlegelstein,Daniel]|[[7,4],[8,8],[9]]|1.300000
-[0:4]|7|Elizabeth|1|False|20|4.700000|1980-10-26|1976-12-23 11:21:42|48:24:11|[2]|[Ein]|[[6],[7],[8]]|1.463000
-[0:5]|8|Farooq|2|True|25|4.500000|1980-10-26|1972-07-31 13:22:30.678559|00:18:00.024|[3,4,5,6,7]|[Fesdwe]|[[8]]|1.510000
-[0:6]|9|Greg|2|False|40|4.900000|1980-10-26|1976-12-23 04:41:42|10 years 5 months 13:00:00.000024|[1]|[Grad]|[[10]]|1.600000
-[0:7]|10|Hubert Blaine Wolfeschlegelsteinhausenbergerdorff|2|False|83|4.900000|1990-11-27|2023-02-21 13:25:30|3 years 2 days 13:02:00|[10,11,12,3,4,5,6,7]|[Ad,De,Hi,Kye,Orlan]|[[7],[10],[6,7]]|1.323000
+[0:0]|0|Alice|1|True|False|35|5.000000|1900-01-01|2011-08-20 11:25:30|3 years 2 days 13:02:00|[10,5]|[Aida]|[[10,8],[6,7,8]]|[96,54,86,92]|1.731000
+[0:1]|2|Bob|2|True|False|30|5.100000|1900-01-01|2008-11-03 15:25:30.000526|10 years 5 months 13:00:00.000024|[12,8]|[Bobby]|[[8,9],[9,10]]|[98,42,93,88]|0.990000
+[0:2]|3|Carol|1|False|True|45|5.000000|1940-06-22|1911-08-20 02:32:21|48:24:11|[4,5]|[Carmen,Fred]|[[8,10]]|[91,75,21,95]|1.000000
+[0:3]|5|Dan|2|False|True|20|4.800000|1950-07-23|2031-11-30 12:25:30|10 years 5 months 13:00:00.000024|[1,9]|[Wolfeschlegelstein,Daniel]|[[7,4],[8,8],[9]]|[76,88,99,89]|1.300000
+[0:4]|7|Elizabeth|1|False|True|20|4.700000|1980-10-26|1976-12-23 11:21:42|48:24:11|[2]|[Ein]|[[6],[7],[8]]|[96,59,65,88]|1.463000
+[0:5]|8|Farooq|2|True|False|25|4.500000|1980-10-26|1972-07-31 13:22:30.678559|00:18:00.024|[3,4,5,6,7]|[Fesdwe]|[[8]]|[80,78,34,83]|1.510000
+[0:6]|9|Greg|2|False|False|40|4.900000|1980-10-26|1976-12-23 04:41:42|10 years 5 months 13:00:00.000024|[1]|[Grad]|[[10]]|[43,83,67,43]|1.600000
+[0:7]|10|Hubert Blaine Wolfeschlegelsteinhausenbergerdorff|2|False|True|83|4.900000|1990-11-27|2023-02-21 13:25:30|3 years 2 days 13:02:00|[10,11,12,3,4,5,6,7]|[Ad,De,Hi,Kye,Orlan]|[[7],[10],[6,7]]|[77,64,100,54]|1.323000
 
--STATEMENT COPY (MATCH (m:movies) RETURN m.name, m.length, m.note, m.description, m.audience) TO "${DATABASE_PATH}/movies.csv"
+-STATEMENT COPY (MATCH (m:movies) RETURN m.*) TO "${DATABASE_PATH}/movies.csv"
 ---- ok
 -STATEMENT load from "${DATABASE_PATH}/movies.csv" return *
 ---- 3
-Sóló cón tu párejâ|126| this is a very very good movie|{rating: 5.300000, stars: 2, views: 152, release: 2011-08-20 11:25:30, film: 2012-05-11, u8: 220, u16: 20, u32: 1, u64: 180, hugedata: 1844674407370955161811111111}|{audience1=52, audience53=42}
-The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie|2544| the movie is very very good|{rating: 7.000000, stars: 10, views: 982, release: 2018-11-13 13:33:11, film: 2014-09-12, u8: 12, u16: 120, u32: 55, u64: 1, hugedata: -1844674407370955161511}|{audience1=33}
-Roma|298|the movie is very interesting and funny|{rating: 1223.000000, stars: 100, views: 10003, release: 2011-02-11 16:44:22, film: 2013-02-22, u8: 1, u16: 15, u32: 200, u64: 4, hugedata: -15}|{}
+Sóló cón tu párejâ|126| this is a very very good movie|{rating: 5.300000, stars: 2, views: 152, release: 2011-08-20 11:25:30, film: 2012-05-11, u8: 220, u16: 20, u32: 1, u64: 180, hugedata: 1844674407370955161811111111}|\xAA\xABinteresting\x0B|{audience1=52, audience53=42}|True
+The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie|2544| the movie is very very good|{rating: 7.000000, stars: 10, views: 982, release: 2018-11-13 13:33:11, film: 2014-09-12, u8: 12, u16: 120, u32: 55, u64: 1, hugedata: -1844674407370955161511}|\xAB\xCD|{audience1=33}|8.989000
+Roma|298|the movie is very interesting and funny|{rating: 1223.000000, stars: 100, views: 10003, release: 2011-02-11 16:44:22, film: 2013-02-22, u8: 1, u16: 15, u32: 200, u64: 4, hugedata: -15}|pure ascii characters|{}|254.000000
 
 
 -STATEMENT COPY (MATCH (p:person)-[s:studyAt]->(o:organisation) RETURN p.ID, s.level, s.places, o.ID) TO "${DATABASE_PATH}/studyAt.csv"
@@ -35,24 +35,24 @@ Roma|298|the movie is very interesting and funny|{rating: 1223.000000, stars: 10
 2|120|[anew,jsdnwusklklklwewsd]|1
 8|2|[awndsnjwejwen,isuhuwennjnuhuhuwewe]|1
 
--STATEMENT COPY (MATCH (p:person)-[e:knows]->(p1:person) RETURN p.ID, p1.ID) TO "${DATABASE_PATH}/onehop.csv"
+-STATEMENT COPY (MATCH (p:person)-[e:knows]->(p1:person) RETURN p.ID, p1.ID, p.grades, p1.grades) TO "${DATABASE_PATH}/onehop.csv"
 ---- ok
 -STATEMENT load from "${DATABASE_PATH}/onehop.csv" return *
 ---- 14
-0|2
-0|3
-0|5
-2|0
-2|3
-2|5
-3|0
-3|2
-3|5
-5|0
-5|2
-5|3
-7|8
-7|9
+0|2|[96,54,86,92]|[98,42,93,88]
+0|3|[96,54,86,92]|[91,75,21,95]
+0|5|[96,54,86,92]|[76,88,99,89]
+2|0|[98,42,93,88]|[96,54,86,92]
+2|3|[98,42,93,88]|[91,75,21,95]
+2|5|[98,42,93,88]|[76,88,99,89]
+3|0|[91,75,21,95]|[96,54,86,92]
+3|2|[91,75,21,95]|[98,42,93,88]
+3|5|[91,75,21,95]|[76,88,99,89]
+5|0|[76,88,99,89]|[96,54,86,92]
+5|2|[76,88,99,89]|[98,42,93,88]
+5|3|[76,88,99,89]|[91,75,21,95]
+7|8|[96,59,65,88]|[80,78,34,83]
+7|9|[96,59,65,88]|[43,83,67,43]
 
 -CASE CopyToWithNullAndEmptyList
 -STATEMENT COPY (RETURN NULL,[],[1,3,NULL,5],[[2,3],[],NULL,[1,5,6]]) TO "${DATABASE_PATH}/nullAndEmptyList.csv"

test/test_files/tinysnb/agg/simple.test

@@ -6,11 +6,11 @@
 -CASE AggSimple
 
 -LOG OneHopSimpleAggTest
--STATEMENT MATCH (a:person)-[:knows]->(b:person)-[:knows]->(c:person) RETURN COUNT(a.ID), MIN(a.fName), MAX(c.ID)
+-STATEMENT MATCH (a:person)-[:knows]->(b:person)-[:knows]->(c:person) RETURN COUNT(a.ID), MIN(a.fName), MAX(c.ID),count(a.grades)
 -PARALLELISM 8
 -ENUMERATE
 ---- 1
-36|Alice|5
+36|Alice|5|36
 
 -LOG SimpleAvgTest
 -STATEMENT MATCH (a:person) RETURN AVG(a.age), AVG(a.eyeSight)
@@ -250,11 +250,11 @@ False
 82
 
 -LOG SimpleAggAvgInt16Test
--STATEMENT MATCH (m:movies) RETURN AVG(m.length)
+-STATEMENT MATCH (m:movies) RETURN AVG(m.length), count(m.audience), count(m.grade)
 -PARALLELISM 7
 -ENUMERATE
 ---- 1
-989.333333
+989.333333|3|3
 
 -LOG SimpleAggMaxInt8Test
 -STATEMENT MATCH (:person)-[s:studyAt]->(:organisation) RETURN MAX(s.level)