Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve duckdb scanner extension #3338

Merged
merged 2 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion extension/duckdb_scanner/test/test_files/duckdb_scanner.test
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ Binder exception: No database named tinysnb has been attached.
-STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/extension/duckdb_scanner/test/duckdb_database/tinysnb.db' (dbtype 'duckdb');
---- 1
Attach database successfully.
-STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/extension/duckdb_scanner/test/duckdb_database/other.db' as other1 (dbtype 'duckdb');
-STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/extension/duckdb_scanner/test/duckdb_database/other.db' as Other1 (dbtype 'duckdb');
---- 1
Attach database successfully.
-STATEMENT LOAD FROM other1.person RETURN *;
Expand Down Expand Up @@ -103,3 +103,7 @@ Runtime exception: No database named NONEXIST.
---- 2
Jeff|32
Wendy|28
-LOG AttachSameDBError
-STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/extension/duckdb_scanner/test/duckdb_database/dbfilewithoutext' (dbtype 'duckdb');
---- error
Runtime exception: Database with name: dbfilewithoutext has already been attached.
12 changes: 12 additions & 0 deletions src/common/string_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ std::vector<std::string> StringUtils::splitBySpace(const std::string& input) {
return result;
}

std::string StringUtils::getUpper(const std::string& input) {
auto result = input;
toUpper(result);
return result;
}

std::string StringUtils::getLower(const std::string& input) {
auto result = input;
toLower(result);
return result;
}

void StringUtils::removeCStringWhiteSpaces(const char*& input, uint64_t& len) {
// skip leading/trailing spaces
while (len > 0 && isspace(input[0])) {
Expand Down
4 changes: 2 additions & 2 deletions src/extension/extension.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ bool ExtensionUtils::isFullPath(const std::string& extension) {
}

ExtensionRepoInfo ExtensionUtils::getExtensionRepoInfo(const std::string& extension) {
auto extensionURL =
common::stringFormat(EXTENSION_REPO, KUZU_EXTENSION_VERSION, getPlatform(), extension);
auto extensionURL = common::stringFormat(EXTENSION_REPO, KUZU_EXTENSION_VERSION, getPlatform(),
common::StringUtils::getLower(extension));
common::StringUtils::replaceAll(extensionURL, "http://", "");
auto hostNamePos = extensionURL.find('/');
auto hostName = extensionURL.substr(0, hostNamePos);
Expand Down
2 changes: 1 addition & 1 deletion src/include/binder/bound_extension_statement.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class BoundExtensionStatement final : public BoundStatement {
public:
explicit BoundExtensionStatement(ExtensionAction action, std::string path)
: BoundStatement{common::StatementType::EXTENSION,
BoundStatementResult::createEmptyResult()},
BoundStatementResult::createSingleStringColumnResult()},
action{action}, path{std::move(path)} {}

inline ExtensionAction getAction() const { return action; }
Expand Down
8 changes: 2 additions & 6 deletions src/include/common/string_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@ class StringUtils {
static void toUpper(std::string& input) {
std::transform(input.begin(), input.end(), input.begin(), ::toupper);
}
static std::string getUpper(const std::string& input) {
auto result = input;
toUpper(result);
return result;
}

static std::string getUpper(const std::string& input);
static std::string getLower(const std::string& input);
static void toLower(std::string& input) {
std::transform(input.begin(), input.end(), input.begin(), ::tolower);
}
Expand Down
34 changes: 0 additions & 34 deletions src/include/planner/operator/logical_extension.h

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class LogicalAttachDatabase final : public LogicalSimple {
public:
explicit LogicalAttachDatabase(parser::AttachInfo attachInfo,
std::shared_ptr<binder::Expression> outputExpression)
: LogicalSimple{LogicalOperatorType::ATTACH_DATABASE, outputExpression},
: LogicalSimple{LogicalOperatorType::ATTACH_DATABASE, std::move(outputExpression)},
attachInfo{std::move(attachInfo)} {}

parser::AttachInfo getAttachInfo() const { return attachInfo; }
Expand Down
33 changes: 33 additions & 0 deletions src/include/planner/operator/simple/logical_extension.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#pragma once

#include "extension/extension_action.h"
#include "logical_simple.h"

namespace kuzu {
namespace planner {

using namespace kuzu::extension;

class LogicalExtension final : public LogicalSimple {
public:
LogicalExtension(ExtensionAction action, std::string path,
std::shared_ptr<binder::Expression> outputExpression)
: LogicalSimple{LogicalOperatorType::EXTENSION, std::move(outputExpression)},
action{action}, path{std::move(path)} {}

std::string getExpressionsForPrinting() const override { return path; }

ExtensionAction getAction() const { return action; }
std::string getPath() const { return path; }

std::unique_ptr<LogicalOperator> copy() override {
return std::make_unique<LogicalExtension>(action, path, outputExpression);
}

private:
ExtensionAction action;
std::string path;
};

} // namespace planner
} // namespace kuzu
36 changes: 0 additions & 36 deletions src/include/processor/operator/install_extension.h

This file was deleted.

33 changes: 0 additions & 33 deletions src/include/processor/operator/load_extension.h

This file was deleted.

4 changes: 2 additions & 2 deletions src/include/processor/operator/simple/attach_database.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ class AttachDatabase final : public Simple {
: Simple{PhysicalOperatorType::ATTACH_DATABASE, outputPos, id, paramsString},
attachInfo{std::move(attachInfo)} {}

void executeInternal(ExecutionContext* context) final;
std::string getOutputMsg() final;
void executeInternal(ExecutionContext* context) override;
std::string getOutputMsg() override;

std::unique_ptr<PhysicalOperator> clone() override {
return std::make_unique<AttachDatabase>(attachInfo, outputPos, id, paramsString);
Expand Down
34 changes: 34 additions & 0 deletions src/include/processor/operator/simple/install_extension.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#pragma once

#include "processor/operator/simple/simple.h"

namespace kuzu {
namespace processor {

class InstallExtension final : public Simple {
public:
InstallExtension(std::string name, const DataPos& outputPos, uint32_t id,
const std::string& paramsString)
: Simple{PhysicalOperatorType::INSTALL_EXTENSION, outputPos, id, paramsString},
name{std::move(name)} {}

void executeInternal(ExecutionContext* context) override;
std::string getOutputMsg() override;

std::unique_ptr<PhysicalOperator> clone() override {
return std::make_unique<InstallExtension>(name, outputPos, id, paramsString);
}

private:
std::string tryDownloadExtension();

void saveExtensionToLocalFile(const std::string& extensionData, main::ClientContext* context);

void installExtension(main::ClientContext* context);

private:
std::string name;
};

} // namespace processor
} // namespace kuzu
27 changes: 27 additions & 0 deletions src/include/processor/operator/simple/load_extension.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#pragma once

#include "processor/operator/simple/simple.h"

namespace kuzu {
namespace processor {

class LoadExtension final : public Simple {
public:
LoadExtension(std::string path, const DataPos& outputPos, uint32_t id,
const std::string& paramsString)
: Simple{PhysicalOperatorType::LOAD_EXTENSION, outputPos, id, paramsString},
path{std::move(path)} {}

void executeInternal(ExecutionContext* context) override;
std::string getOutputMsg() override;

std::unique_ptr<PhysicalOperator> clone() override {
return std::make_unique<LoadExtension>(path, outputPos, id, paramsString);
}

private:
std::string path;
};

} // namespace processor
} // namespace kuzu
4 changes: 4 additions & 0 deletions src/main/database_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ void DatabaseManager::registerAttachedDatabase(std::unique_ptr<AttachedDatabase>
if (defaultDatabase == "") {
defaultDatabase = attachedDatabase->getDBName();
}
if (getAttachedDatabase(attachedDatabase->getDBName()) != nullptr) {
throw common::RuntimeException{common::stringFormat(
"Database with name: {} has already been attached.", attachedDatabase->getDBName())};
}
attachedDatabases.push_back(std::move(attachedDatabase));
}

Expand Down
4 changes: 2 additions & 2 deletions src/planner/plan/append_simple.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
#include "planner/operator/logical_comment_on.h"
#include "planner/operator/logical_create_macro.h"
#include "planner/operator/logical_explain.h"
#include "planner/operator/logical_extension.h"
#include "planner/operator/logical_standalone_call.h"
#include "planner/operator/logical_transaction.h"
#include "planner/operator/simple/logical_attach_database.h"
#include "planner/operator/simple/logical_detach_database.h"
#include "planner/operator/simple/logical_extension.h"
#include "planner/operator/simple/logical_use_database.h"
#include "planner/planner.h"

Expand Down Expand Up @@ -98,7 +98,7 @@ void Planner::appendExtension(const BoundStatement& statement, LogicalPlan& plan
auto& extensionStatement =
common::ku_dynamic_cast<const BoundStatement&, const BoundExtensionStatement&>(statement);
auto op = std::make_shared<LogicalExtension>(extensionStatement.getAction(),
extensionStatement.getPath());
extensionStatement.getPath(), statement.getStatementResult()->getSingleColumnExpr());
plan.setLastOperator(std::move(op));
}

Expand Down
1 change: 0 additions & 1 deletion src/processor/map/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ add_library(kuzu_processor_mapper
map_distinct.cpp
map_explain.cpp
map_expressions_scan.cpp
map_extension.cpp
map_dummy_scan.cpp
map_empty_result.cpp
map_extend.cpp
Expand Down
29 changes: 0 additions & 29 deletions src/processor/map/map_extension.cpp

This file was deleted.

Loading
Loading