Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove bm resize interface #1382

Merged
merged 1 commit into from
Mar 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions src/include/main/database.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,6 @@ class Database {
*/
static void setLoggingLevel(std::string loggingLevel);

/**
* @brief Resizes the buffer pool size of the database instance.
* @param newSize New buffer pool size in bytes.
* @throws BufferManagerException if the new size is smaller than the current buffer manager
* size.
*/
KUZU_API void resizeBufferManager(uint64_t newSize);

// Temporary patching for C-style APIs.
// TODO(Change): move this to C-header once we have C-APIs.
KUZU_API explicit Database(const char* databasePath);
Expand Down
2 changes: 0 additions & 2 deletions src/include/storage/buffer_manager/buffer_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@ class BufferManager {
bufferPoolDefaultPages->unpinWithoutAcquiringPageLock(fileHandle, pageIdx);
}

void resize(uint64_t newSizeForDefaultPagePool, uint64_t newSizeForLargePagePool);

void removeFilePagesFromFrames(BufferManagedFileHandle& fileHandle);

void flushAllDirtyPagesInFrames(BufferManagedFileHandle& fileHandle);
Expand Down
2 changes: 0 additions & 2 deletions src/include/storage/buffer_manager/buffer_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,6 @@ class BufferPool {
void unpinWithoutAcquiringPageLock(
BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx);

void resize(uint64_t newSize);

// Note: These two functions that remove pages from frames is not designed for concurrency and
// therefore not tested under concurrency. If this is called while other threads are accessing
// the BM, it should work safely but this is not tested.
Expand Down
7 changes: 0 additions & 7 deletions src/main/database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,6 @@ void Database::setLoggingLevel(std::string loggingLevel) {
spdlog::set_level(LoggingLevelUtils::convertStrToLevelEnum(std::move(loggingLevel)));
}

void Database::resizeBufferManager(uint64_t newSize) {
systemConfig.defaultPageBufferPoolSize = newSize * StorageConstants::DEFAULT_PAGES_BUFFER_RATIO;
systemConfig.largePageBufferPoolSize = newSize * StorageConstants::LARGE_PAGES_BUFFER_RATIO;
bufferManager->resize(
systemConfig.defaultPageBufferPoolSize, systemConfig.largePageBufferPoolSize);
}

void Database::commitAndCheckpointOrRollback(
Transaction* writeTransaction, bool isCommit, bool skipCheckpointForTestingRecovery) {
// Irrespective of whether we are checkpointing or rolling back we add a
Expand Down
5 changes: 0 additions & 5 deletions src/storage/buffer_manager/buffer_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@ BufferManager::BufferManager(uint64_t maxSizeForDefaultPagePool, uint64_t maxSiz

BufferManager::~BufferManager() = default;

void BufferManager::resize(uint64_t newSizeForDefaultPagePool, uint64_t newSizeForLargePagePool) {
bufferPoolDefaultPages->resize(newSizeForDefaultPagePool);
bufferPoolLargePages->resize(newSizeForLargePagePool);
}

// Important Note: Pin returns a raw pointer to the frame. This is potentially very dangerous and
// trusts the caller is going to protect this memory space.
// Important responsibilities for the caller are:
Expand Down
17 changes: 0 additions & 17 deletions src/storage/buffer_manager/buffer_pool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,23 +68,6 @@ BufferPool::BufferPool(uint64_t pageSize, uint64_t maxSize)
pageSize, numFrames);
}

void BufferPool::resize(uint64_t newSize) {
if ((numFrames * pageSize) > newSize) {
throw BufferManagerException("Resizing to a smaller Buffer Pool Size is unsupported.");
}
auto newNumFrames = (page_idx_t)(ceil((double)newSize / (double)pageSize));
assert(newNumFrames < UINT32_MAX);
auto mmapRegion =
(uint8_t*)mmap(NULL, newSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
for (auto i = 0u; i < newNumFrames - numFrames; ++i) {
auto buffer = mmapRegion + (i * pageSize);
bufferCache.emplace_back(std::make_unique<Frame>(pageSize, buffer));
}
numFrames = newNumFrames;
logger->info(
"Resize buffer pool's max size to {}B, #{}byte-pages {}.", newSize, pageSize, newNumFrames);
}

uint8_t* BufferPool::pin(BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) {
return pin(fileHandle, pageIdx, false /* read page from file */);
}
Expand Down
10 changes: 0 additions & 10 deletions test/main/config_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,6 @@
using namespace kuzu::common;
using namespace kuzu::testing;

TEST_F(ApiTest, DatabaseConfig) {
spdlog::set_level(spdlog::level::debug);
ASSERT_NO_THROW(database->resizeBufferManager(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2));
ASSERT_EQ(
getDefaultBMSize(*database), (uint64_t)(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2 *
StorageConstants::DEFAULT_PAGES_BUFFER_RATIO));
ASSERT_EQ(getLargeBMSize(*database), (uint64_t)(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2 *
StorageConstants::LARGE_PAGES_BUFFER_RATIO));
}

TEST_F(ApiTest, ClientConfig) {
spdlog::set_level(spdlog::level::debug);
ASSERT_NO_THROW(conn->setMaxNumThreadForExec(2));
Expand Down
2 changes: 0 additions & 2 deletions tools/python_api/src_cpp/include/py_database.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ class PyDatabase {

explicit PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize);

void resizeBufferManager(uint64_t newSize);

~PyDatabase() = default;

private:
Expand Down
5 changes: 0 additions & 5 deletions tools/python_api/src_cpp/py_database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ void PyDatabase::initialize(py::handle& m) {
py::class_<PyDatabase>(m, "Database")
.def(py::init<const std::string&, uint64_t>(), py::arg("database_path"),
py::arg("buffer_pool_size") = 0)
.def("resize_buffer_manager", &PyDatabase::resizeBufferManager, py::arg("new_size"))
.def("set_logging_level", &PyDatabase::setLoggingLevel, py::arg("logging_level"));
}

Expand All @@ -20,7 +19,3 @@ PyDatabase::PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize)
}
database = std::make_unique<Database>(databasePath, systemConfig);
}

void PyDatabase::resizeBufferManager(uint64_t newSize) {
database->resizeBufferManager(newSize);
}
7 changes: 0 additions & 7 deletions tools/python_api/test/test_df.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,6 @@ def _test_to_df(conn):
conn.set_max_threads_for_exec(1)
_test_to_df(conn)

db.resize_buffer_manager(384 * 1024 * 1024)
_test_to_df(conn)

db.resize_buffer_manager(512 * 1024 * 1024)
conn.set_max_threads_for_exec(4)
_test_to_df(conn)

db.set_logging_level("debug")
_test_to_df(conn)

Expand Down
4 changes: 0 additions & 4 deletions tools/python_api/test/test_exception.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@ def test_exception(establish_connection):
with pytest.raises(RuntimeError, match="Binder exception: Cannot find property dummy for a."):
conn.execute("MATCH (a:person) RETURN a.dummy;")

with pytest.raises(RuntimeError,
match="Buffer manager exception: Resizing to a smaller Buffer Pool Size is unsupported."):
db.resize_buffer_manager(1)


def test_db_path_exception():
path = '/:* /? " < > |'
Expand Down
21 changes: 2 additions & 19 deletions tools/shell/embedded_shell.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,13 @@ struct ShellCommand {
const std::string CLEAR = ":clear";
const std::string QUIT = ":quit";
const std::string THREAD = ":thread";
const std::string BUFFER_MANAGER_SIZE = ":buffer_manager_size";
const std::string LIST_NODES = ":list_nodes";
const std::string LIST_RELS = ":list_rels";
const std::string SHOW_NODE = ":show_node";
const std::string SHOW_REL = ":show_rel";
const std::string LOGGING_LEVEL = ":logging_level";
const std::vector<std::string> commandList = {HELP, CLEAR, QUIT, THREAD, BUFFER_MANAGER_SIZE,
LIST_NODES, LIST_RELS, SHOW_NODE, SHOW_REL, LOGGING_LEVEL};
const std::vector<std::string> commandList = {HELP, CLEAR, QUIT, THREAD, LIST_NODES, LIST_RELS,
SHOW_NODE, SHOW_REL, LOGGING_LEVEL};
} shellCommand;

const char* TAB = " ";
Expand Down Expand Up @@ -227,8 +226,6 @@ void EmbeddedShell::run() {
break;
} else if (lineStr.rfind(shellCommand.THREAD) == 0) {
setNumThreads(lineStr.substr(shellCommand.THREAD.length()));
} else if (lineStr.rfind(shellCommand.BUFFER_MANAGER_SIZE) == 0) {
setBufferManagerSize(lineStr.substr(shellCommand.BUFFER_MANAGER_SIZE.length()));
} else if (lineStr.rfind(shellCommand.LIST_NODES) == 0) {
printf("%s", conn->getNodeTableNames().c_str());
} else if (lineStr.rfind(shellCommand.LIST_RELS) == 0) {
Expand Down Expand Up @@ -278,18 +275,6 @@ void EmbeddedShell::setNumThreads(const std::string& numThreadsString) {
} catch (Exception& e) { printf("%s", e.what()); }
}

void EmbeddedShell::setBufferManagerSize(const std::string& bufferManagerSizeString) {
auto newPageSize = 0;
try {
newPageSize = stoull(bufferManagerSizeString);
} catch (std::exception& e) {
printf("Cannot parse '%s' as buffer manager size. Expect integer.\n", e.what());
}
try {
database->resizeBufferManager(newPageSize);
} catch (Exception& e) { printf("%s\n", e.what()); }
}

static inline std::string ltrim(const std::string& input) {
auto s = input;
s.erase(s.begin(), find_if(s.begin(), s.end(), [](unsigned char ch) { return !isspace(ch); }));
Expand All @@ -316,8 +301,6 @@ void EmbeddedShell::printHelp() {
printf("%s%s %sexit from shell\n", TAB, shellCommand.QUIT.c_str(), TAB);
printf("%s%s [num_threads] %sset number of threads for query execution\n", TAB,
shellCommand.THREAD.c_str(), TAB);
printf("%s%s [bm_size_in_bytes] %sset buffer manager size in bytes\n", TAB,
shellCommand.BUFFER_MANAGER_SIZE.c_str(), TAB);
printf("%s%s [logging_level] %sset logging level of database, available options: debug, info, "
"err\n",
TAB, shellCommand.LOGGING_LEVEL.c_str(), TAB);
Expand Down
2 changes: 0 additions & 2 deletions tools/shell/include/embedded_shell.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ class EmbeddedShell {
private:
void setNumThreads(const std::string& numThreadsString);

void setBufferManagerSize(const std::string& bufferManagerSizeString);

void printNodeSchema(const std::string& tableName);
void printRelSchema(const std::string& tableName);

Expand Down