-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1396 from kuzudb/copy-npy-1
Copy node tables from npy files
- Loading branch information
Showing
64 changed files
with
4,622 additions
and
3,120 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npytable fromnpy ("dataset/npy-1d/one_dim_int64.npy", "dataset/npy-1d/one_dim_int32.npy", "dataset/npy-1d/one_dim_int16.npy", "dataset/npy-1d/one_dim_double.npy", "dataset/npy-1d/one_dim_float.npy"); |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npytable (i64 INT64,i32 INT32,i16 INT16,f64 DOUBLE,f32 FLOAT, PRIMARY KEY(i64)); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npytable fromNPY ("dataset/npy-20k/id_int64.npy", "dataset/npy-20k/two_dim_float.npy"); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npytable (id INT64,f32 FLOAT[10],PRIMARY KEY(id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npytable fromNPY ("dataset/npy-2d/id_int64.npy", "dataset/npy-2d/two_dim_int64.npy", "dataset/npy-2d/two_dim_int32.npy", "dataset/npy-2d/two_dim_int16.npy", "dataset/npy-2d/two_dim_double.npy", "dataset/npy-2d/two_dim_float.npy"); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npytable (id INT64, i64 INT64[3],i32 INT32[3],i16 INT16[3],f64 DOUBLE[3],f32 FLOAT[3],PRIMARY KEY(id)); |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
copy npytable FROMnpy ("dataset/npy-3d/id_int64.npy", "dataset/npy-3d/three_dim_int64.npy"); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
create node table npytable (id INT64,i64 INT64[12],PRIMARY KEY(id)); |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
#pragma once | ||
|
||
#include "common/copier_config/copier_config.h" | ||
#include "common/logging_level_utils.h" | ||
#include "common/task_system/task_scheduler.h" | ||
#include "node_copier.h" | ||
#include "npy_reader.h" | ||
#include "storage/in_mem_storage_structure/in_mem_column.h" | ||
#include "storage/index/hash_index_builder.h" | ||
#include "storage/store/nodes_statistics_and_deleted_ids.h" | ||
|
||
using namespace kuzu::catalog; | ||
using namespace kuzu::common; | ||
|
||
namespace kuzu { | ||
namespace storage { | ||
|
||
class CopyNodeNpy : public NodeCopier { | ||
|
||
public: | ||
CopyNodeNpy(CopyDescription& copyDescription, std::string outputDirectory, | ||
TaskScheduler& taskScheduler, catalog::Catalog& catalog, table_id_t tableID, | ||
NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs) | ||
: NodeCopier(copyDescription, outputDirectory, taskScheduler, catalog, tableID, | ||
nodesStatisticsAndDeletedIDs){}; | ||
|
||
~CopyNodeNpy() = default; | ||
|
||
private: | ||
void populateColumnsAndLists() override; | ||
|
||
void populateInMemoryStructures() override; | ||
|
||
void initializeNpyReaders(); | ||
|
||
void validateNpyReaders(); | ||
|
||
void populateColumnsFromNpy(std::unique_ptr<HashIndexBuilder<int64_t>>& pkIndex); | ||
|
||
void assignCopyNpyTasks( | ||
common::property_id_t propertyIdx, std::unique_ptr<HashIndexBuilder<int64_t>>& pkIndex); | ||
|
||
static void batchPopulateColumnsTask(common::property_id_t primaryKeyPropertyIdx, | ||
uint64_t blockIdx, offset_t startOffset, uint64_t numLinesInCurBlock, | ||
HashIndexBuilder<int64_t>* pkIndex, CopyNodeNpy* copier, common::property_id_t propertyIdx); | ||
|
||
private: | ||
std::unordered_map<common::property_id_t, std::unique_ptr<NpyReader>> npyReaderMap; | ||
}; | ||
|
||
} // namespace storage | ||
} // namespace kuzu |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
Oops, something went wrong.