-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
32 changed files
with
50,855 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,3 +12,4 @@ build/ | |
__pycache__/ | ||
*.py[cod] | ||
*$py.class | ||
cmake-build-debug/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
COPY tableOfTypes FROM "dataset/copy-test/node/arrow/types_50k.arrow" (HEADER=true); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table tableOfTypes (id INT64, int64Column INT64, doubleColumn DOUBLE, booleanColumn BOOLEAN, dateColumn DATE, timestampColumn TIMESTAMP, stringColumn STRING, PRIMARY KEY (id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
COPY tableOfTypes FROM "dataset/copy-test/node/csv/types_50k.csv" (HEADER=true); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table tableOfTypes (id INT64, int64Column INT64, doubleColumn DOUBLE, booleanColumn BOOLEAN, dateColumn DATE, timestampColumn TIMESTAMP, stringColumn STRING, PRIMARY KEY (id)); |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
COPY tableOfTypes FROM "dataset/copy-test/node/parquet/types_50k.parquet" (HEADER=true); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table tableOfTypes (id INT64, int64Column INT64, doubleColumn DOUBLE, booleanColumn BOOLEAN, dateColumn DATE, timestampColumn TIMESTAMP, stringColumn STRING, PRIMARY KEY (id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,9 @@ | ||
id,fname,Gender,ISStudent,isWorker,age,eyeSight,birthdate,registerTime,lastJobDuration,workedHours,usedNames,courseScoresPerTerm | ||
0,Alice,1,true,false,35,5.0,1900-01-01,2011-08-20 11:25:30Z+00:00,3 years 2 days 13 hours 2 minutes,[10,5],["Aida"],[[10,8],[6,7,8]] | ||
2,Bob,2,true,false,30,5.1,1900-01-01,2008-11-03 13:25:30.000526-02:00,10 years 5 months 13 hours 24 us,[12,8],[Bobby],[[8,9],[9,10]] | ||
3,Carol,1,false,true,45,5.0,1940-06-22,1911-08-20 02:32:21,48 hours 24 minutes 11 seconds,[4,5],[Carmen,Fred],[[8,10]] | ||
5,Dan,2,false,true,20,4.8,1950-7-23,2031-11-30 12:25:30Z,10 years 5 months 13 hours 24 us,[1,9],[Wolfeschlegelstein,Daniel],[[7,4],[8,8],[9]] | ||
7,Elizabeth,1,false,true,20,4.7,1980-10-26,1976-12-23 11:21:42,48 hours 24 minutes 11 seconds,[2],[Ein],[[6],[7],[8]] | ||
8,Farooq,2,true,false,25,4.5,1980-10-26,1972-07-31 13:22:30.678559,18 minutes 24 milliseconds,[3,4,5,6,7],[Fesdwe],[[8]] | ||
9,Greg,2,false,false,40,4.9,1980-10-26,1976-12-23 11:21:42Z+06:40,10 years 5 months 13 hours 24 us,[1],[Grad],[[10]] | ||
10,Hubert Blaine Wolfeschlegelsteinhausenbergerdorff,2,false,true,83,4.9,1990-11-27,2023-02-21 13:25:30,3 years 2 days 13 hours 2 minutes,[10,11,12,3,4,5,6,7],[Ad,De,Hi,Kye,Orlan],[[7],[10],[6,7]] | ||
0,Alice,1,true,false,35,5.0,1900-01-01,2011-08-20 11:25:30Z+00:00,3 years 2 days 13 hours 2 minutes,"[10,5]","[Aida]","[[10,8],[6,7,8]]" | ||
2,Bob,2,true,false,30,5.1,1900-01-01,2008-11-03 13:25:30.000526-02:00,10 years 5 months 13 hours 24 us,"[12,8]","[Bobby]","[[8,9],[9,10]]" | ||
3,Carol,1,false,true,45,5.0,1940-06-22,1911-08-20 02:32:21,48 hours 24 minutes 11 seconds,"[4,5]","[Carmen,Fred]","[[8,10]]" | ||
5,Dan,2,false,true,20,4.8,1950-7-23,2031-11-30 12:25:30Z,10 years 5 months 13 hours 24 us,"[1,9]","[Wolfeschlegelstein,Daniel]","[[7,4],[8,8],[9]]" | ||
7,Elizabeth,1,false,true,20,4.7,1980-10-26,1976-12-23 11:21:42,48 hours 24 minutes 11 seconds,"[2]","[Ein]","[[6],[7],[8]]" | ||
8,Farooq,2,true,false,25,4.5,1980-10-26,1972-07-31 13:22:30.678559,18 minutes 24 milliseconds,"[3,4,5,6,7]","[Fesdwe]","[[8]]" | ||
9,Greg,2,false,false,40,4.9,1980-10-26,1976-12-23 11:21:42Z+06:40,10 years 5 months 13 hours 24 us,"[1]","[Grad]","[[10]]" | ||
10,Hubert Blaine Wolfeschlegelsteinhausenbergerdorff,2,false,true,83,4.9,1990-11-27,2023-02-21 13:25:30,3 years 2 days 13 hours 2 minutes,"[10,11,12,3,4,5,6,7]","[Ad,De,Hi,Kye,Orlan]","[[7],[10],[6,7]]" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
104 changes: 104 additions & 0 deletions
104
src/include/storage/in_mem_csv_copier/in_mem_arrow_node_copier.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
#pragma once | ||
|
||
#include "in_mem_structures_copier.h" | ||
#include "storage/index/hash_index_builder.h" | ||
#include "storage/store/nodes_statistics_and_deleted_ids.h" | ||
#include <arrow/api.h> | ||
#include <arrow/csv/api.h> | ||
#include <arrow/io/api.h> | ||
#include <arrow/ipc/reader.h> | ||
#include <arrow/pretty_print.h> | ||
#include <arrow/result.h> | ||
#include <arrow/scalar.h> | ||
#include <arrow/status.h> | ||
#include <arrow/table.h> | ||
#include <parquet/arrow/reader.h> | ||
#include <parquet/arrow/writer.h> | ||
#include <parquet/exception.h> | ||
|
||
namespace kuzu { | ||
namespace storage { | ||
|
||
class InMemArrowNodeCopier : public InMemStructuresCopier { | ||
|
||
public: | ||
InMemArrowNodeCopier(CSVDescription& csvDescription, string outputDirectory, | ||
TaskScheduler& taskScheduler, Catalog& catalog, table_id_t tableID, | ||
NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs); | ||
|
||
~InMemArrowNodeCopier() override = default; | ||
|
||
uint64_t copy(); | ||
|
||
void saveToFile() override; | ||
|
||
enum class FileTypes { CSV, ARROW, PARQUET }; | ||
|
||
static std::string getFileTypeName(FileTypes fileTypes); | ||
|
||
static std::string getFileTypeSuffix(FileTypes fileTypes); | ||
|
||
private: | ||
void setFileType(std::string const& fileName); | ||
|
||
void countNumLines(const std::string& filePath); | ||
|
||
arrow::Status countNumLinesCSV(std::string const& filePath); | ||
|
||
arrow::Status countNumLinesArrow(std::string const& filePath); | ||
|
||
arrow::Status countNumLinesParquet(std::string const& filePath); | ||
|
||
void initializeColumnsAndList(); | ||
|
||
template<typename T> | ||
arrow::Status populateColumns(); | ||
|
||
template<typename T> | ||
arrow::Status populateColumnsFromCSV(unique_ptr<HashIndexBuilder<T>>& pkIndex); | ||
|
||
template<typename T> | ||
arrow::Status populateColumnsFromArrow(unique_ptr<HashIndexBuilder<T>>& pkIndex); | ||
|
||
template<typename T> | ||
arrow::Status populateColumnsFromParquet(unique_ptr<HashIndexBuilder<T>>& pkIndex); | ||
|
||
template<typename T> | ||
static void putPropsOfLineIntoColumns(vector<unique_ptr<InMemColumn>>& columns, | ||
vector<PageByteCursor>& overflowCursors, const std::vector<shared_ptr<T>>& arrow_columns, | ||
uint64_t nodeOffset, uint64_t bufferOffset, char delimiter); | ||
|
||
template<typename T> | ||
static void populatePKIndex(InMemColumn* column, HashIndexBuilder<T>* pkIndex, | ||
node_offset_t startOffset, uint64_t numValues); | ||
|
||
// Concurrent tasks. | ||
// Note that primaryKeyPropertyIdx is *NOT* the property ID of the primary key property. | ||
|
||
template<typename T1, typename T2> | ||
static arrow::Status batchPopulateColumnsTask(uint64_t primaryKeyPropertyIdx, uint64_t blockId, | ||
uint64_t offsetStart, HashIndexBuilder<T1>* pkIndex, InMemArrowNodeCopier* copier, | ||
const vector<shared_ptr<T2>>& batchColumns, char delimiter); | ||
|
||
arrow::Status initCSVReader( | ||
shared_ptr<arrow::csv::StreamingReader>& csv_streaming_reader, const std::string& filePath); | ||
|
||
arrow::Status initArrowReader(std::shared_ptr<arrow::ipc::RecordBatchFileReader>& ipc_reader, | ||
const std::string& filePath); | ||
|
||
arrow::Status initParquetReader( | ||
std::unique_ptr<parquet::arrow::FileReader>& reader, const std::string& filePath); | ||
|
||
static Literal getArrowList( | ||
string& l, int64_t from, int64_t to, const DataType& dataType, char delimiter); | ||
|
||
private: | ||
NodeTableSchema* nodeTableSchema; | ||
uint64_t numNodes; | ||
vector<unique_ptr<InMemColumn>> columns; | ||
NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs; | ||
FileTypes inputFileType; | ||
}; | ||
|
||
} // namespace storage | ||
} // namespace kuzu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.