-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2329 from kuzudb/parquet-copy
Parquet copy improvement
- Loading branch information
Showing
79 changed files
with
34,845 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
src/include/processor/operator/persistent/reader/parquet/interval_column_reader.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
#pragma once | ||
|
||
#include "common/types/interval_t.h" | ||
#include "templated_column_reader.h" | ||
|
||
namespace kuzu { | ||
namespace processor { | ||
|
||
struct IntervalValueConversion { | ||
static inline common::interval_t dictRead( | ||
ByteBuffer& dict, uint32_t& offset, ColumnReader& /*reader*/) { | ||
return (reinterpret_cast<common::interval_t*>(dict.ptr))[offset]; | ||
} | ||
|
||
static common::interval_t readParquetInterval(const char* input); | ||
|
||
static common::interval_t plainRead(ByteBuffer& plainData, ColumnReader& reader); | ||
|
||
static inline void plainSkip(ByteBuffer& plain_data, ColumnReader& /*reader*/) { | ||
plain_data.inc(common::ParquetConstants::PARQUET_INTERVAL_SIZE); | ||
} | ||
}; | ||
|
||
class IntervalColumnReader | ||
: public TemplatedColumnReader<common::interval_t, IntervalValueConversion> { | ||
|
||
public: | ||
IntervalColumnReader(ParquetReader& reader, std::unique_ptr<common::LogicalType> type, | ||
const kuzu_parquet::format::SchemaElement& schema, uint64_t fileIdx, uint64_t maxDefine, | ||
uint64_t maxRepeat) | ||
: TemplatedColumnReader<common::interval_t, IntervalValueConversion>( | ||
reader, std::move(type), schema, fileIdx, maxDefine, maxRepeat){}; | ||
|
||
protected: | ||
void dictionary( | ||
std::shared_ptr<ResizeableBuffer> dictionary_data, uint64_t num_entries) override; | ||
}; | ||
|
||
} // namespace processor | ||
} // namespace kuzu |
32 changes: 32 additions & 0 deletions
32
src/include/processor/operator/persistent/writer/parquet/interval_column_writer.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#pragma once | ||
|
||
#include "basic_column_writer.h" | ||
#include "common/types/interval_t.h" | ||
|
||
namespace kuzu { | ||
namespace processor { | ||
|
||
class IntervalColumnWriter : public BasicColumnWriter { | ||
|
||
public: | ||
IntervalColumnWriter(ParquetWriter& writer, uint64_t schemaIdx, | ||
std::vector<std::string> schemaPath, uint64_t maxRepeat, uint64_t maxDefine, | ||
bool canHaveNulls) | ||
: BasicColumnWriter( | ||
writer, schemaIdx, std::move(schemaPath), maxRepeat, maxDefine, canHaveNulls) {} | ||
|
||
public: | ||
static void writeParquetInterval(common::interval_t input, uint8_t* result); | ||
|
||
void writeVector(common::Serializer& bufferedSerializer, ColumnWriterStatistics* state, | ||
ColumnWriterPageState* pageState, common::ValueVector* vector, uint64_t chunkStart, | ||
uint64_t chunkEnd) override; | ||
|
||
inline uint64_t getRowSize(common::ValueVector* /*vector*/, uint64_t /*index*/, | ||
BasicColumnWriterState& /*state*/) override { | ||
return common::ParquetConstants::PARQUET_INTERVAL_SIZE; | ||
} | ||
}; | ||
|
||
} // namespace processor | ||
} // namespace kuzu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
src/processor/operator/persistent/reader/parquet/interval_column_reader.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#include "processor/operator/persistent/reader/parquet/interval_column_reader.h" | ||
|
||
namespace kuzu { | ||
namespace processor { | ||
|
||
common::interval_t IntervalValueConversion::readParquetInterval(const char* input) { | ||
common::interval_t result; | ||
auto inputData = reinterpret_cast<const uint32_t*>(input); | ||
result.months = inputData[0]; | ||
result.days = inputData[1]; | ||
result.micros = int64_t(inputData[2]) * 1000; | ||
return result; | ||
} | ||
|
||
common::interval_t IntervalValueConversion::plainRead( | ||
ByteBuffer& plainData, ColumnReader& /*reader*/) { | ||
auto intervalLen = common::ParquetConstants::PARQUET_INTERVAL_SIZE; | ||
plainData.available(intervalLen); | ||
auto res = readParquetInterval(reinterpret_cast<const char*>(plainData.ptr)); | ||
plainData.inc(intervalLen); | ||
return res; | ||
} | ||
|
||
void IntervalColumnReader::dictionary( | ||
std::shared_ptr<ResizeableBuffer> dictionary_data, uint64_t num_entries) { | ||
allocateDict(num_entries * sizeof(common::interval_t)); | ||
auto dict_ptr = reinterpret_cast<common::interval_t*>(this->dict->ptr); | ||
for (auto i = 0u; i < num_entries; i++) { | ||
dict_ptr[i] = IntervalValueConversion::plainRead(*dictionary_data, *this); | ||
} | ||
} | ||
|
||
} // namespace processor | ||
} // namespace kuzu |
Oops, something went wrong.