Skip to content

Commit

Permalink
remove bm resize interface
Browse files Browse the repository at this point in the history
  • Loading branch information
ray6080 committed Mar 15, 2023
1 parent e3961e6 commit f67e3cf
Show file tree
Hide file tree
Showing 13 changed files with 2 additions and 90 deletions.
8 changes: 0 additions & 8 deletions src/include/main/database.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,6 @@ class Database {
*/
static void setLoggingLevel(std::string loggingLevel);

/**
* @brief Resizes the buffer pool size of the database instance.
* @param newSize New buffer pool size in bytes.
* @throws BufferManagerException if the new size is smaller than the current buffer manager
* size.
*/
KUZU_API void resizeBufferManager(uint64_t newSize);

// Temporary patching for C-style APIs.
// TODO(Change): move this to C-header once we have C-APIs.
KUZU_API explicit Database(const char* databasePath);
Expand Down
2 changes: 0 additions & 2 deletions src/include/storage/buffer_manager/buffer_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@ class BufferManager {
bufferPoolDefaultPages->unpinWithoutAcquiringPageLock(fileHandle, pageIdx);
}

void resize(uint64_t newSizeForDefaultPagePool, uint64_t newSizeForLargePagePool);

void removeFilePagesFromFrames(BufferManagedFileHandle& fileHandle);

void flushAllDirtyPagesInFrames(BufferManagedFileHandle& fileHandle);
Expand Down
2 changes: 0 additions & 2 deletions src/include/storage/buffer_manager/buffer_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,6 @@ class BufferPool {
void unpinWithoutAcquiringPageLock(
BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx);

void resize(uint64_t newSize);

// Note: These two functions that remove pages from frames is not designed for concurrency and
// therefore not tested under concurrency. If this is called while other threads are accessing
// the BM, it should work safely but this is not tested.
Expand Down
7 changes: 0 additions & 7 deletions src/main/database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,6 @@ void Database::setLoggingLevel(std::string loggingLevel) {
spdlog::set_level(LoggingLevelUtils::convertStrToLevelEnum(std::move(loggingLevel)));
}

void Database::resizeBufferManager(uint64_t newSize) {
systemConfig.defaultPageBufferPoolSize = newSize * StorageConstants::DEFAULT_PAGES_BUFFER_RATIO;
systemConfig.largePageBufferPoolSize = newSize * StorageConstants::LARGE_PAGES_BUFFER_RATIO;
bufferManager->resize(
systemConfig.defaultPageBufferPoolSize, systemConfig.largePageBufferPoolSize);
}

void Database::commitAndCheckpointOrRollback(
Transaction* writeTransaction, bool isCommit, bool skipCheckpointForTestingRecovery) {
// Irrespective of whether we are checkpointing or rolling back we add a
Expand Down
5 changes: 0 additions & 5 deletions src/storage/buffer_manager/buffer_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@ BufferManager::BufferManager(uint64_t maxSizeForDefaultPagePool, uint64_t maxSiz

BufferManager::~BufferManager() = default;

void BufferManager::resize(uint64_t newSizeForDefaultPagePool, uint64_t newSizeForLargePagePool) {
bufferPoolDefaultPages->resize(newSizeForDefaultPagePool);
bufferPoolLargePages->resize(newSizeForLargePagePool);
}

// Important Note: Pin returns a raw pointer to the frame. This is potentially very dangerous and
// trusts the caller is going to protect this memory space.
// Important responsibilities for the caller are:
Expand Down
17 changes: 0 additions & 17 deletions src/storage/buffer_manager/buffer_pool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,23 +68,6 @@ BufferPool::BufferPool(uint64_t pageSize, uint64_t maxSize)
pageSize, numFrames);
}

void BufferPool::resize(uint64_t newSize) {
if ((numFrames * pageSize) > newSize) {
throw BufferManagerException("Resizing to a smaller Buffer Pool Size is unsupported.");
}
auto newNumFrames = (page_idx_t)(ceil((double)newSize / (double)pageSize));
assert(newNumFrames < UINT32_MAX);
auto mmapRegion =
(uint8_t*)mmap(NULL, newSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
for (auto i = 0u; i < newNumFrames - numFrames; ++i) {
auto buffer = mmapRegion + (i * pageSize);
bufferCache.emplace_back(std::make_unique<Frame>(pageSize, buffer));
}
numFrames = newNumFrames;
logger->info(
"Resize buffer pool's max size to {}B, #{}byte-pages {}.", newSize, pageSize, newNumFrames);
}

uint8_t* BufferPool::pin(BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) {
return pin(fileHandle, pageIdx, false /* read page from file */);
}
Expand Down
10 changes: 0 additions & 10 deletions test/main/config_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,6 @@
using namespace kuzu::common;
using namespace kuzu::testing;

TEST_F(ApiTest, DatabaseConfig) {
spdlog::set_level(spdlog::level::debug);
ASSERT_NO_THROW(database->resizeBufferManager(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2));
ASSERT_EQ(
getDefaultBMSize(*database), (uint64_t)(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2 *
StorageConstants::DEFAULT_PAGES_BUFFER_RATIO));
ASSERT_EQ(getLargeBMSize(*database), (uint64_t)(StorageConstants::DEFAULT_BUFFER_POOL_SIZE * 2 *
StorageConstants::LARGE_PAGES_BUFFER_RATIO));
}

TEST_F(ApiTest, ClientConfig) {
spdlog::set_level(spdlog::level::debug);
ASSERT_NO_THROW(conn->setMaxNumThreadForExec(2));
Expand Down
2 changes: 0 additions & 2 deletions tools/python_api/src_cpp/include/py_database.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ class PyDatabase {

explicit PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize);

void resizeBufferManager(uint64_t newSize);

~PyDatabase() = default;

private:
Expand Down
5 changes: 0 additions & 5 deletions tools/python_api/src_cpp/py_database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ void PyDatabase::initialize(py::handle& m) {
py::class_<PyDatabase>(m, "Database")
.def(py::init<const std::string&, uint64_t>(), py::arg("database_path"),
py::arg("buffer_pool_size") = 0)
.def("resize_buffer_manager", &PyDatabase::resizeBufferManager, py::arg("new_size"))
.def("set_logging_level", &PyDatabase::setLoggingLevel, py::arg("logging_level"));
}

Expand All @@ -20,7 +19,3 @@ PyDatabase::PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize)
}
database = std::make_unique<Database>(databasePath, systemConfig);
}

void PyDatabase::resizeBufferManager(uint64_t newSize) {
database->resizeBufferManager(newSize);
}
7 changes: 0 additions & 7 deletions tools/python_api/test/test_df.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,6 @@ def _test_to_df(conn):
conn.set_max_threads_for_exec(1)
_test_to_df(conn)

db.resize_buffer_manager(384 * 1024 * 1024)
_test_to_df(conn)

db.resize_buffer_manager(512 * 1024 * 1024)
conn.set_max_threads_for_exec(4)
_test_to_df(conn)

db.set_logging_level("debug")
_test_to_df(conn)

Expand Down
4 changes: 0 additions & 4 deletions tools/python_api/test/test_exception.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@ def test_exception(establish_connection):
with pytest.raises(RuntimeError, match="Binder exception: Cannot find property dummy for a."):
conn.execute("MATCH (a:person) RETURN a.dummy;")

with pytest.raises(RuntimeError,
match="Buffer manager exception: Resizing to a smaller Buffer Pool Size is unsupported."):
db.resize_buffer_manager(1)


def test_db_path_exception():
path = '/:* /? " < > |'
Expand Down
21 changes: 2 additions & 19 deletions tools/shell/embedded_shell.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,13 @@ struct ShellCommand {
const std::string CLEAR = ":clear";
const std::string QUIT = ":quit";
const std::string THREAD = ":thread";
const std::string BUFFER_MANAGER_SIZE = ":buffer_manager_size";
const std::string LIST_NODES = ":list_nodes";
const std::string LIST_RELS = ":list_rels";
const std::string SHOW_NODE = ":show_node";
const std::string SHOW_REL = ":show_rel";
const std::string LOGGING_LEVEL = ":logging_level";
const std::vector<std::string> commandList = {HELP, CLEAR, QUIT, THREAD, BUFFER_MANAGER_SIZE,
LIST_NODES, LIST_RELS, SHOW_NODE, SHOW_REL, LOGGING_LEVEL};
const std::vector<std::string> commandList = {HELP, CLEAR, QUIT, THREAD, LIST_NODES, LIST_RELS,
SHOW_NODE, SHOW_REL, LOGGING_LEVEL};
} shellCommand;

const char* TAB = " ";
Expand Down Expand Up @@ -227,8 +226,6 @@ void EmbeddedShell::run() {
break;
} else if (lineStr.rfind(shellCommand.THREAD) == 0) {
setNumThreads(lineStr.substr(shellCommand.THREAD.length()));
} else if (lineStr.rfind(shellCommand.BUFFER_MANAGER_SIZE) == 0) {
setBufferManagerSize(lineStr.substr(shellCommand.BUFFER_MANAGER_SIZE.length()));
} else if (lineStr.rfind(shellCommand.LIST_NODES) == 0) {
printf("%s", conn->getNodeTableNames().c_str());
} else if (lineStr.rfind(shellCommand.LIST_RELS) == 0) {
Expand Down Expand Up @@ -278,18 +275,6 @@ void EmbeddedShell::setNumThreads(const std::string& numThreadsString) {
} catch (Exception& e) { printf("%s", e.what()); }
}

void EmbeddedShell::setBufferManagerSize(const std::string& bufferManagerSizeString) {
auto newPageSize = 0;
try {
newPageSize = stoull(bufferManagerSizeString);
} catch (std::exception& e) {
printf("Cannot parse '%s' as buffer manager size. Expect integer.\n", e.what());
}
try {
database->resizeBufferManager(newPageSize);
} catch (Exception& e) { printf("%s\n", e.what()); }
}

static inline std::string ltrim(const std::string& input) {
auto s = input;
s.erase(s.begin(), find_if(s.begin(), s.end(), [](unsigned char ch) { return !isspace(ch); }));
Expand All @@ -316,8 +301,6 @@ void EmbeddedShell::printHelp() {
printf("%s%s %sexit from shell\n", TAB, shellCommand.QUIT.c_str(), TAB);
printf("%s%s [num_threads] %sset number of threads for query execution\n", TAB,
shellCommand.THREAD.c_str(), TAB);
printf("%s%s [bm_size_in_bytes] %sset buffer manager size in bytes\n", TAB,
shellCommand.BUFFER_MANAGER_SIZE.c_str(), TAB);
printf("%s%s [logging_level] %sset logging level of database, available options: debug, info, "
"err\n",
TAB, shellCommand.LOGGING_LEVEL.c_str(), TAB);
Expand Down
2 changes: 0 additions & 2 deletions tools/shell/include/embedded_shell.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ class EmbeddedShell {
private:
void setNumThreads(const std::string& numThreadsString);

void setBufferManagerSize(const std::string& bufferManagerSizeString);

void printNodeSchema(const std::string& tableName);
void printRelSchema(const std::string& tableName);

Expand Down

0 comments on commit f67e3cf

Please sign in to comment.