diff --git a/src/include/storage/store/column_chunk.h b/src/include/storage/store/column_chunk.h index 6181310822..8c2852723b 100644 --- a/src/include/storage/store/column_chunk.h +++ b/src/include/storage/store/column_chunk.h @@ -93,9 +93,6 @@ class ColumnChunk { virtual void append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - virtual void append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - ColumnChunkMetadata flushBuffer( BMFileHandle* dataFH, common::page_idx_t startPageIdx, const ColumnChunkMetadata& metadata); @@ -140,22 +137,6 @@ class ColumnChunk { // Initializes the data buffer. Is (and should be) only called in constructor. virtual void initialize(common::offset_t capacity); - template - void templateCopyArrowArray( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - template - void templateCopyStringArrowArray( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - // TODO(Guodong/Ziyi): The conversion from string to values should be handled inside ReadFile. - // ARROW_TYPE can be either arrow::StringArray or arrow::LargeStringArray. - template - void templateCopyValuesAsString( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - - virtual inline common::page_idx_t getNumPagesForBuffer() const { - return getNumPagesForBytes(bufferSize); - } - common::offset_t getOffsetInBuffer(common::offset_t pos) const; virtual void copyVectorToBuffer(common::ValueVector* vector, common::offset_t startPosInChunk); @@ -199,9 +180,6 @@ class BoolColumnChunk : public ColumnChunk { void append(common::ValueVector* vector, common::offset_t startPosInChunk) final; - void append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; - void append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend) override; @@ -288,12 +266,6 @@ struct ColumnChunkFactory { const common::LogicalType& dataType, common::CSVReaderConfig* csvReaderConfig = nullptr); }; -template<> -void ColumnChunk::templateCopyArrowArray( - arrow::Array* array, common::offset_t startPosInSegment, uint32_t numValuesToAppend); -template<> -void ColumnChunk::templateCopyArrowArray( - arrow::Array* array, common::offset_t startPosInSegment, uint32_t numValuesToAppend); // BOOL template<> void ColumnChunk::setValueFromString(const char* value, uint64_t length, uint64_t pos); diff --git a/src/include/storage/store/string_column_chunk.h b/src/include/storage/store/string_column_chunk.h index b98f03ef17..a29e348d66 100644 --- a/src/include/storage/store/string_column_chunk.h +++ b/src/include/storage/store/string_column_chunk.h @@ -14,8 +14,6 @@ class StringColumnChunk : public ColumnChunk { void resetToEmpty() final; void append(common::ValueVector* vector, common::offset_t startPosInChunk) final; - void append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; void append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; @@ -36,13 +34,6 @@ class StringColumnChunk : public ColumnChunk { inline common::offset_t getLastOffsetInPage() { return overflowCursor.offsetInPage; } private: - template - void templateCopyStringArrowArray( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - template - void templateCopyStringValues( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - void appendStringColumnChunk(StringColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend); diff --git a/src/include/storage/store/struct_column_chunk.h b/src/include/storage/store/struct_column_chunk.h index aab081d33f..d502793c43 100644 --- a/src/include/storage/store/struct_column_chunk.h +++ b/src/include/storage/store/struct_column_chunk.h @@ -11,8 +11,6 @@ class StructColumnChunk : public ColumnChunk { common::LogicalType dataType, std::unique_ptr csvReaderConfig); protected: - void append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; void append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; void append(common::ValueVector* vector, common::offset_t startPosInChunk) final; @@ -24,11 +22,6 @@ class StructColumnChunk : public ColumnChunk { void setValueToStructField(common::offset_t pos, const std::string& structFieldValue, common::struct_field_idx_t structFiledIdx); void write(const common::Value& val, uint64_t posToWrite) final; - void copyStructFromArrowStruct( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - template - void copyStructFromArrowString( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); }; } // namespace storage diff --git a/src/include/storage/store/var_list_column_chunk.h b/src/include/storage/store/var_list_column_chunk.h index d11f955d1b..45d969b1ba 100644 --- a/src/include/storage/store/var_list_column_chunk.h +++ b/src/include/storage/store/var_list_column_chunk.h @@ -52,35 +52,9 @@ class VarListColumnChunk : public ColumnChunk { } private: - void append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) override; - void append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend) final; - void copyVarListFromArrowString( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend); - - template - void copyVarListFromArrowList( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto listArray = (T*)array; - auto dataChunkOffsetToAppend = varListDataColumnChunk.getNumValues(); - auto curListOffset = varListDataColumnChunk.getNumValues(); - for (auto i = 0u; i < numValuesToAppend; i++) { - nullChunk->setNull(i + startPosInChunk, listArray->IsNull(i)); - auto length = listArray->value_length(i); - curListOffset += length; - setValue(curListOffset, i + startPosInChunk); - } - auto startOffset = listArray->value_offset(0); - auto endOffset = listArray->value_offset(numValuesToAppend); - varListDataColumnChunk.resizeBuffer(curListOffset); - varListDataColumnChunk.dataColumnChunk->append( - listArray->values().get(), dataChunkOffsetToAppend, endOffset - startOffset); - numValues += numValuesToAppend; - } - void write(const common::Value& listVal, uint64_t posToWrite) override; private: diff --git a/src/processor/operator/persistent/copy_node.cpp b/src/processor/operator/persistent/copy_node.cpp index 950741e542..ce1e4b8953 100644 --- a/src/processor/operator/persistent/copy_node.cpp +++ b/src/processor/operator/persistent/copy_node.cpp @@ -110,21 +110,14 @@ void CopyNode::executeInternal(ExecutionContext* context) { void CopyNode::sliceDataChunk( const DataChunk& dataChunk, const std::vector& dataColumnPoses, offset_t offset) { - if (dataChunk.valueVectors[0]->dataType.getPhysicalType() == PhysicalTypeID::ARROW_COLUMN) { - for (auto& dataColumnPos : dataColumnPoses) { - ArrowColumnVector::slice( - dataChunk.valueVectors[dataColumnPos.valueVectorPos].get(), offset); - } - } else { - auto slicedSelVector = std::make_unique(DEFAULT_VECTOR_CAPACITY); - slicedSelVector->resetSelectorToValuePosBufferWithSize( - dataChunk.state->selVector->selectedSize - offset); - for (auto i = 0u; i < slicedSelVector->selectedSize; i++) { - slicedSelVector->selectedPositions[i] = - dataChunk.state->selVector->selectedPositions[i + offset]; - } - dataChunk.state->selVector = std::move(slicedSelVector); + auto slicedSelVector = std::make_unique(DEFAULT_VECTOR_CAPACITY); + slicedSelVector->resetSelectorToValuePosBufferWithSize( + dataChunk.state->selVector->selectedSize - offset); + for (auto i = 0u; i < slicedSelVector->selectedSize; i++) { + slicedSelVector->selectedPositions[i] = + dataChunk.state->selVector->selectedPositions[i + offset]; } + dataChunk.state->selVector = std::move(slicedSelVector); } void CopyNode::writeAndResetNodeGroup(node_group_idx_t nodeGroupIdx, diff --git a/src/storage/store/column_chunk.cpp b/src/storage/store/column_chunk.cpp index 99f692536f..b5bbd07e3c 100644 --- a/src/storage/store/column_chunk.cpp +++ b/src/storage/store/column_chunk.cpp @@ -1,6 +1,5 @@ #include "storage/store/column_chunk.h" -#include "arrow/array.h" #include "common/types/value/nested.h" #include "storage/storage_structure/storage_structure_utils.h" #include "storage/store/compression.h" @@ -212,21 +211,7 @@ void ColumnChunk::append(common::ValueVector* vector, common::offset_t startPosI void ColumnChunk::append( ValueVector* vector, offset_t startPosInChunk, uint32_t numValuesToAppend) { - if (vector->dataType.getPhysicalType() == PhysicalTypeID::ARROW_COLUMN) { - auto chunkedArray = ArrowColumnVector::getArrowColumn(vector).get(); - for (const auto& array : chunkedArray->chunks()) { - auto numValuesInArrayToAppend = - std::min((uint64_t)array->length(), (uint64_t)numValuesToAppend); - if (numValuesInArrayToAppend <= 0) { - break; - } - append(array.get(), startPosInChunk, numValuesInArrayToAppend); - numValuesToAppend -= numValuesInArrayToAppend; - startPosInChunk += numValuesInArrayToAppend; - } - } else { - append(vector, startPosInChunk); - } + append(vector, startPosInChunk); } void ColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChunk, @@ -241,62 +226,6 @@ void ColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChunk, numValues += numValuesToAppend; } -void ColumnChunk::append( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - switch (array->type_id()) { - case arrow::Type::INT8: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::INT16: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::INT32: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::INT64: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::UINT8: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::UINT16: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::UINT32: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::UINT64: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::DOUBLE: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::FLOAT: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::DATE32: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::TIMESTAMP: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::FIXED_SIZE_LIST: { - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::STRING: { - templateCopyStringArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::LARGE_STRING: { - templateCopyStringArrowArray( - array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("ColumnChunk::append"); - } - } - numValues += numValuesToAppend; -} - void ColumnChunk::write(const Value& val, uint64_t posToWrite) { nullChunk->setNull(posToWrite, val.isNull()); if (val.isNull()) { @@ -381,127 +310,6 @@ void ColumnChunk::populateWithDefaultVal(common::ValueVector* defaultValueVector } } -template -void ColumnChunk::templateCopyArrowArray( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - const auto& arrowArray = array->data(); - auto valuesInChunk = (T*)buffer.get(); - auto valuesInArray = arrowArray->GetValues(1 /* value buffer */); - if (arrowArray->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrowArray->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - valuesInChunk[posInChunk] = valuesInArray[i]; - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - valuesInChunk[posInChunk] = valuesInArray[i]; - } - } -} - -template -void ColumnChunk::templateCopyStringArrowArray( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { - switch (dataType.getLogicalTypeID()) { - case LogicalTypeID::DATE: { - templateCopyValuesAsString(array, startPosInChunk, numValuesToAppend); - } break; - case LogicalTypeID::TIMESTAMP: { - templateCopyValuesAsString( - array, startPosInChunk, numValuesToAppend); - } break; - case LogicalTypeID::INTERVAL: { - templateCopyValuesAsString( - array, startPosInChunk, numValuesToAppend); - } break; - case LogicalTypeID::FIXED_LIST: { - // Fixed list is a fixed-sized blob. - templateCopyValuesAsString(array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("ColumnChunk::templateCopyStringArrowArray"); - } - } -} - -template<> -void ColumnChunk::templateCopyArrowArray( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto* boolArray = (arrow::BooleanArray*)array; - auto data = boolArray->data(); - - auto arrowBuffer = boolArray->values()->data(); - // Might read off the end with the cast, but copyNullMask should ignore the extra data - // - // The arrow BooleanArray offset should be the offset in bits - // Unfortunately this is not documented. - NullMask::copyNullMask((uint64_t*)arrowBuffer, boolArray->offset(), (uint64_t*)buffer.get(), - startPosInChunk, numValuesToAppend); - - if (data->MayHaveNulls()) { - auto arrowNullBitMap = boolArray->null_bitmap_data(); - - // Offset should apply to both bool data and nulls - nullChunk->copyFromBuffer((uint64_t*)arrowNullBitMap, boolArray->offset(), startPosInChunk, - numValuesToAppend, true /*invert*/); - } -} - -template<> -void ColumnChunk::templateCopyArrowArray( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto fixedSizedListArray = (arrow::FixedSizeListArray*)array; - auto valuesInList = (uint8_t*)fixedSizedListArray->values()->data()->buffers[1]->data(); - if (fixedSizedListArray->data()->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (fixedSizedListArray->data()->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - auto posInList = fixedSizedListArray->offset() + i; - memcpy(buffer.get() + getOffsetInBuffer(posInChunk), - valuesInList + posInList * numBytesPerValue, numBytesPerValue); - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - auto posInList = fixedSizedListArray->offset() + i; - memcpy(buffer.get() + getOffsetInBuffer(posInChunk), - valuesInList + posInList * numBytesPerValue, numBytesPerValue); - } - } -} - -template -void ColumnChunk::templateCopyValuesAsString( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto stringArray = (ARROW_TYPE*)array; - auto arrayData = stringArray->data(); - if (arrayData->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrayData->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - auto value = stringArray->GetView(i); - setValueFromString(value.data(), value.length(), posInChunk); - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - auto value = stringArray->GetView(i); - setValueFromString(value.data(), value.length(), posInChunk); - } - } -} - ColumnChunkMetadata ColumnChunk::getMetadataToFlush() const { return getMetadataFunction(buffer.get(), bufferSize, capacity, numValues); } @@ -550,13 +358,6 @@ void BoolColumnChunk::append(common::ValueVector* vector, common::offset_t start numValues += vector->state->selVector->selectedSize; } -void BoolColumnChunk::append( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { - assert(array->type_id() == arrow::Type::BOOL); - templateCopyArrowArray(array, startPosInChunk, numValuesToAppend); - numValues += numValuesToAppend; -} - void BoolColumnChunk::append(ColumnChunk* other, common::offset_t startPosInOtherChunk, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { NullMask::copyNullMask((uint64_t*)static_cast(other)->buffer.get(), @@ -633,7 +434,6 @@ void FixedListColumnChunk::copyVectorToBuffer( for (auto i = 0u; i < vector->state->selVector->selectedSize; i++) { auto pos = vector->state->selVector->selectedPositions[i]; nullChunk->setNull(startPosInChunk + i, vector->isNull(pos)); - auto offset = getOffsetInBuffer(startPosInChunk + i); memcpy(buffer.get() + getOffsetInBuffer(startPosInChunk + i), vectorDataToWriteFrom + pos * numBytesPerValue, numBytesPerValue); } diff --git a/src/storage/store/string_column_chunk.cpp b/src/storage/store/string_column_chunk.cpp index ef0c676ac0..b40f88dbab 100644 --- a/src/storage/store/string_column_chunk.cpp +++ b/src/storage/store/string_column_chunk.cpp @@ -38,25 +38,6 @@ void StringColumnChunk::append(common::ValueVector* vector, common::offset_t sta numValues += vector->state->selVector->selectedSize; } -void StringColumnChunk::append( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - assert( - array->type_id() == arrow::Type::STRING || array->type_id() == arrow::Type::LARGE_STRING); - switch (array->type_id()) { - case arrow::Type::STRING: { - templateCopyStringArrowArray(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::LARGE_STRING: { - templateCopyStringArrowArray( - array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("StringColumnChunk::append"); - } - } - numValues += numValuesToAppend; -} - void StringColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChunk, offset_t startPosInChunk, uint32_t numValuesToAppend) { auto otherChunk = reinterpret_cast(other); @@ -161,45 +142,5 @@ std::string StringColumnChunk::getValue(offset_t pos) const { return overflowFile->readString(&kuStr); } -template -void StringColumnChunk::templateCopyStringArrowArray( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { - switch (dataType.getLogicalTypeID()) { - case LogicalTypeID::BLOB: { - templateCopyStringValues(array, startPosInChunk, numValuesToAppend); - } break; - case LogicalTypeID::STRING: { - templateCopyStringValues(array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("StringColumnChunk::templateCopyStringArrowArray"); - } - } -} - -template -void StringColumnChunk::templateCopyStringValues( - arrow::Array* array, common::offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto stringArray = (ARROW_TYPE*)array; - auto arrayData = stringArray->data(); - if (arrayData->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrayData->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - auto value = stringArray->GetView(i); - setValueFromString(value.data(), value.length(), posInChunk); - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - auto value = stringArray->GetView(i); - setValueFromString(value.data(), value.length(), posInChunk); - } - } -} - } // namespace storage } // namespace kuzu diff --git a/src/storage/store/struct_column_chunk.cpp b/src/storage/store/struct_column_chunk.cpp index 5048cbb6d6..3b78b6fd8c 100644 --- a/src/storage/store/struct_column_chunk.cpp +++ b/src/storage/store/struct_column_chunk.cpp @@ -1,6 +1,5 @@ #include "storage/store/struct_column_chunk.h" -#include "common/exception/copy.h" #include "common/exception/not_implemented.h" #include "common/exception/parser.h" #include "common/string_utils.h" @@ -25,27 +24,6 @@ StructColumnChunk::StructColumnChunk( } } -void StructColumnChunk::append( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - switch (array->type_id()) { - case arrow::Type::STRUCT: { - copyStructFromArrowStruct(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::STRING: { - copyStructFromArrowString(array, startPosInChunk, numValuesToAppend); - - } break; - case arrow::Type::LARGE_STRING: { - copyStructFromArrowString( - array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("StructColumnChunk::append"); - } - } - numValues += numValuesToAppend; -} - void StructColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChunk, offset_t startPosInChunk, uint32_t numValuesToAppend) { auto otherStructChunk = dynamic_cast(other); @@ -196,56 +174,5 @@ void StructColumnChunk::write(const Value& val, uint64_t posToWrite) { } } -void StructColumnChunk::copyStructFromArrowStruct( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto structArray = (arrow::StructArray*)array; - auto arrayData = structArray->data(); - if (StructType::getNumFields(&dataType) != structArray->type()->fields().size()) { - throw CopyException{"Unmatched number of struct fields in StructColumnChunk::append."}; - } - for (auto i = 0u; i < structArray->num_fields(); i++) { - auto fieldName = structArray->type()->fields()[i]->name(); - auto fieldIdx = StructType::getFieldIdx(&dataType, fieldName); - if (fieldIdx == INVALID_STRUCT_FIELD_IDX) { - throw CopyException{"Unmatched struct field name: " + fieldName + "."}; - } - childrenChunks[fieldIdx]->append( - structArray->field(i).get(), startPosInChunk, numValuesToAppend); - } - if (arrayData->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrayData->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - } - } -} - -template -void StructColumnChunk::copyStructFromArrowString( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto* stringArray = (ARROW_TYPE*)array; - auto arrayData = stringArray->data(); - if (arrayData->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrayData->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - continue; - } - auto value = stringArray->GetView(i); - setStructFields(value.data(), value.length(), posInChunk); - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - auto value = stringArray->GetView(i); - setStructFields(value.data(), value.length(), posInChunk); - } - } -} - } // namespace storage } // namespace kuzu diff --git a/src/storage/store/var_list_column_chunk.cpp b/src/storage/store/var_list_column_chunk.cpp index c09a08b617..d870e2514b 100644 --- a/src/storage/store/var_list_column_chunk.cpp +++ b/src/storage/store/var_list_column_chunk.cpp @@ -32,27 +32,6 @@ VarListColumnChunk::VarListColumnChunk( assert(this->dataType.getPhysicalType() == PhysicalTypeID::VAR_LIST); } -void VarListColumnChunk::append( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - assert(array->type_id() == arrow::Type::STRING || array->type_id() == arrow::Type::LIST || - array->type_id() == arrow::Type::LARGE_LIST); - switch (array->type_id()) { - case arrow::Type::STRING: { - copyVarListFromArrowString(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::LIST: { - copyVarListFromArrowList(array, startPosInChunk, numValuesToAppend); - } break; - case arrow::Type::LARGE_LIST: { - copyVarListFromArrowList(array, startPosInChunk, numValuesToAppend); - } break; - default: { - throw NotImplementedException("ListColumnChunk::appendArray"); - } - } - numValues += numValuesToAppend; -} - void VarListColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChunk, offset_t startPosInChunk, uint32_t numValuesToAppend) { nullChunk->append( @@ -72,34 +51,6 @@ void VarListColumnChunk::append(ColumnChunk* other, offset_t startPosInOtherChun numValues += numValuesToAppend; } -void VarListColumnChunk::copyVarListFromArrowString( - arrow::Array* array, offset_t startPosInChunk, uint32_t numValuesToAppend) { - auto stringArray = (arrow::StringArray*)array; - auto arrayData = stringArray->data(); - if (arrayData->MayHaveNulls()) { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto posInChunk = startPosInChunk + i; - if (arrayData->IsNull(i)) { - nullChunk->setNull(posInChunk, true); - setValue(varListDataColumnChunk.getNumValues(), posInChunk); - continue; - } - auto value = stringArray->GetView(i); - auto listVal = TableCopyUtils::getVarListValue( - value.data(), 1, value.size() - 2, dataType, *csvReaderConfig); - write(*listVal, posInChunk); - } - } else { - for (auto i = 0u; i < numValuesToAppend; i++) { - auto value = stringArray->GetView(i); - auto posInChunk = startPosInChunk + i; - auto listVal = TableCopyUtils::getVarListValue( - value.data(), 1, value.size() - 2, dataType, *csvReaderConfig); - write(*listVal, posInChunk); - } - } -} - void VarListColumnChunk::write(const Value& listVal, uint64_t posToWrite) { assert(listVal.getDataType()->getPhysicalType() == PhysicalTypeID::VAR_LIST); auto numValuesInList = NestedVal::getChildrenSize(&listVal);