-
Notifications
You must be signed in to change notification settings - Fork 86
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Refactor copy node #1590
Refactor copy node #1590
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
#pragma once | ||
|
||
#include "arrow/array.h" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's think of a way to get rid of this include. I hope we can go through |
||
#include "common/in_mem_overflow_buffer.h" | ||
|
||
namespace kuzu { | ||
|
@@ -43,6 +44,13 @@ class StructAuxiliaryBuffer : public AuxiliaryBuffer { | |
std::vector<std::shared_ptr<ValueVector>> childrenVectors; | ||
}; | ||
|
||
class ArrowColumnAuxiliaryBuffer : public AuxiliaryBuffer { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the best way to implement the arrow auxiliary is that we store the arrow array in the aux to keep its lifetime, and point to the aux array when possible from ValueVector. Let's discuss this a bit more offline. |
||
friend class ArrowColumnVector; | ||
|
||
private: | ||
std::shared_ptr<arrow::Array> column; | ||
}; | ||
|
||
// ListVector layout: | ||
// To store a list value in the valueVector, we could use two separate vectors. | ||
// 1. A vector(called offset vector) for the list offsets and length(called list_entry_t): This | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
#pragma once | ||
|
||
#include "processor/operator/copy/copy.h" | ||
#include "storage/store/nodes_store.h" | ||
|
||
namespace kuzu { | ||
namespace processor { | ||
|
||
class CopyNPY : public Copy { | ||
acquamarin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
public: | ||
CopyNPY(catalog::Catalog* catalog, const common::CopyDescription& copyDescription, | ||
storage::NodeTable* table, storage::WAL* wal, | ||
storage::NodesStatisticsAndDeletedIDs* nodesStatistics, storage::RelsStore& relsStore, | ||
uint32_t id, const std::string& paramsString) | ||
: Copy{PhysicalOperatorType::COPY_NPY, catalog, copyDescription, table->getTableID(), wal, | ||
id, paramsString}, | ||
table{table}, nodesStatistics{nodesStatistics}, relsStore{relsStore} {} | ||
|
||
std::unique_ptr<PhysicalOperator> clone() override { | ||
return std::make_unique<CopyNPY>( | ||
catalog, copyDescription, table, wal, nodesStatistics, relsStore, id, paramsString); | ||
} | ||
|
||
protected: | ||
uint64_t executeInternal( | ||
common::TaskScheduler* taskScheduler, ExecutionContext* executionContext) override; | ||
|
||
private: | ||
inline bool isCopyAllowed() override { | ||
return nodesStatistics->getNodeStatisticsAndDeletedIDs(tableID)->getNumTuples() == 0; | ||
} | ||
|
||
private: | ||
storage::NodeTable* table; | ||
storage::NodesStatisticsAndDeletedIDs* nodesStatistics; | ||
storage::RelsStore& relsStore; | ||
}; | ||
|
||
} // namespace processor | ||
} // namespace kuzu |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I still don't think this is a good idea. It doesn't make sense to me to have ARROW column/array as a logical type.