-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
56 changed files
with
4,048 additions
and
2,453 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npy(i64,i32,i16,f64,f32) from "dataset/npy-1d/one_dim_int64.npy", "dataset/npy-1d/one_dim_int32.npy", "dataset/npy-1d/one_dim_int16.npy", "dataset/npy-1d/one_dim_double.npy", "dataset/npy-1d/one_dim_float.npy"; |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npy (i64 INT64,i32 INT32,i16 INT16,f64 DOUBLE,f32 FLOAT, PRIMARY KEY(i64)); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npy(id,f32) from "dataset/npy-20k/id_int64.npy", "dataset/npy-20k/two_dim_float.npy"; |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npy (id INT64,f32 FLOAT[10],PRIMARY KEY(id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npy(id,i64,i32,i16,f64,f32) from "dataset/npy-2d/id_int64.npy", "dataset/npy-2d/two_dim_int64.npy", "dataset/npy-2d/two_dim_int32.npy", "dataset/npy-2d/two_dim_int16.npy", "dataset/npy-2d/two_dim_double.npy", "dataset/npy-2d/two_dim_float.npy"; |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npy (id INT64, i64 INT64[3],i32 INT32[3],i16 INT16[3],f64 DOUBLE[3],f32 FLOAT[3],PRIMARY KEY(id)); |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npy(id,i64) from "dataset/npy-3d/id_int64.npy", "dataset/npy-3d/three_dim_int64.npy"; |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npy (id INT64,i64 INT64[12],PRIMARY KEY(id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
#pragma once | ||
|
||
#include "common/copier_config/copier_config.h" | ||
#include "common/logging_level_utils.h" | ||
#include "common/task_system/task_scheduler.h" | ||
#include "npy_reader.h" | ||
#include "storage/in_mem_storage_structure/in_mem_column.h" | ||
#include "storage/index/hash_index_builder.h" | ||
#include "storage/store/nodes_statistics_and_deleted_ids.h" | ||
|
||
using namespace kuzu::catalog; | ||
using namespace kuzu::common; | ||
|
||
namespace kuzu { | ||
namespace storage { | ||
|
||
class CopyNodeNpy { | ||
|
||
public: | ||
CopyNodeNpy(CopyDescription& copyDescription, std::string outputDirectory, | ||
TaskScheduler& taskScheduler, catalog::Catalog& catalog, table_id_t tableID, | ||
NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs) | ||
: logger{LoggerUtils::getLogger(LoggerConstants::LoggerEnum::LOADER)}, | ||
copyDescription{copyDescription}, outputDirectory{std::move(outputDirectory)}, | ||
taskScheduler{taskScheduler}, catalog{catalog}, tableID{tableID}, | ||
nodesStatisticsAndDeletedIDs{nodesStatisticsAndDeletedIDs}, | ||
tableSchema{catalog.getReadOnlyVersion()->getTableSchema(tableID)} {} | ||
|
||
uint64_t copy(); | ||
|
||
~CopyNodeNpy() = default; | ||
|
||
private: | ||
inline void updateTableStatistics() { | ||
nodesStatisticsAndDeletedIDs->setNumTuplesForTable(tableSchema->tableID, numRows); | ||
} | ||
|
||
void initializeColumnsAndLists(); | ||
|
||
void initializeNpyReaders(); | ||
|
||
void validateNpyReaders(); | ||
|
||
void populateColumns(); | ||
|
||
void populateColumnsFromNpy(std::unique_ptr<HashIndexBuilder<int64_t>>& pkIndex); | ||
|
||
void assignCopyNpyTasks( | ||
const std::string& property, std::unique_ptr<HashIndexBuilder<int64_t>>& pkIndex); | ||
|
||
static void batchPopulateColumnsTask(std::string primaryKeyPropertyIdx, uint64_t blockIdx, | ||
offset_t startOffset, uint64_t numLinesInCurBlock, HashIndexBuilder<int64_t>* pkIndex, | ||
CopyNodeNpy* copier, std::string property); | ||
|
||
static void populatePKIndex(InMemColumn* column, HashIndexBuilder<int64_t>* pkIndex, | ||
uint64_t startOffset, uint64_t numValues); | ||
|
||
void saveToFile(); | ||
|
||
private: | ||
std::shared_ptr<spdlog::logger> logger; | ||
CopyDescription copyDescription; | ||
std::string outputDirectory; | ||
TaskScheduler& taskScheduler; | ||
catalog::Catalog& catalog; | ||
table_id_t tableID; | ||
std::unordered_map<std::string, std::unique_ptr<InMemColumn>> columnMap; | ||
std::unordered_map<std::string, std::unique_ptr<NpyReader>> npyReaderMap; | ||
catalog::TableSchema* tableSchema; | ||
uint64_t numRows; | ||
NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs; | ||
}; | ||
|
||
} // namespace storage | ||
} // namespace kuzu |
File renamed without changes.
File renamed without changes.
File renamed without changes.
Oops, something went wrong.