Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename config to constants #1293

Merged
merged 1 commit into from
Feb 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/binder/binder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ void Binder::validateTableExist(const Catalog& _catalog, std::string& tableName)
}

bool Binder::validateStringParsingOptionName(std::string& parsingOptionName) {
for (auto i = 0; i < std::size(CopyConfig::STRING_CSV_PARSING_OPTIONS); i++) {
if (parsingOptionName == CopyConfig::STRING_CSV_PARSING_OPTIONS[i]) {
for (auto i = 0; i < std::size(CopyConstants::STRING_CSV_PARSING_OPTIONS); i++) {
if (parsingOptionName == CopyConstants::STRING_CSV_PARSING_OPTIONS[i]) {
return true;
}
}
Expand Down
19 changes: 10 additions & 9 deletions src/common/csv_reader/csv_reader.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "common/csv_reader/csv_reader.h"

#include "common/configs.h"
#include "common/constants.h"
#include "common/type_utils.h"
#include "common/utils.h"
#include "spdlog/spdlog.h"
Expand All @@ -13,8 +13,8 @@ namespace common {

CSVReader::CSVReader(const std::string& fName, const CSVReaderConfig& config, uint64_t blockId)
: CSVReader{fName, config} {
readingBlockStartOffset = CopyConfig::CSV_READING_BLOCK_SIZE * blockId;
readingBlockEndOffset = CopyConfig::CSV_READING_BLOCK_SIZE * (blockId + 1);
readingBlockStartOffset = CopyConstants::CSV_READING_BLOCK_SIZE * blockId;
readingBlockEndOffset = CopyConstants::CSV_READING_BLOCK_SIZE * (blockId + 1);
auto isBeginningOfLine = false;
if (0 == readingBlockStartOffset) {
isBeginningOfLine = true;
Expand Down Expand Up @@ -233,10 +233,11 @@ uint8_t CSVReader::getBoolean() {
char* CSVReader::getString() {
setNextTokenIsProcessed();
auto strVal = line + linePtrStart;
if (strlen(strVal) > DEFAULT_PAGE_SIZE) {
logger->warn(StringUtils::getLongStringErrorMessage(strVal, DEFAULT_PAGE_SIZE));
if (strlen(strVal) > BufferPoolConstants::DEFAULT_PAGE_SIZE) {
logger->warn(
StringUtils::getLongStringErrorMessage(strVal, BufferPoolConstants::DEFAULT_PAGE_SIZE));
// If the std::string is too long, truncate it.
strVal[DEFAULT_PAGE_SIZE] = '\0';
strVal[BufferPoolConstants::DEFAULT_PAGE_SIZE] = '\0';
}
auto unicodeType = Utf8Proc::analyze(strVal, strlen(strVal));
if (unicodeType == UnicodeType::ASCII) {
Expand Down Expand Up @@ -307,10 +308,10 @@ std::unique_ptr<Value> CSVReader::getList(const DataType& dataType) {
}
}
auto numBytesOfOverflow = listVal.size() * Types::getDataTypeSize(dataType.typeID);
if (numBytesOfOverflow >= DEFAULT_PAGE_SIZE) {
if (numBytesOfOverflow >= BufferPoolConstants::DEFAULT_PAGE_SIZE) {
throw ReaderException(StringUtils::string_format(
"Maximum num bytes of a LIST is %d. Input list's num bytes is %d.", DEFAULT_PAGE_SIZE,
numBytesOfOverflow));
"Maximum num bytes of a LIST is %d. Input list's num bytes is %d.",
BufferPoolConstants::DEFAULT_PAGE_SIZE, numBytesOfOverflow));
}
return std::make_unique<Value>(
DataType(LIST, std::make_unique<DataType>(dataType)), std::move(listVal));
Expand Down
2 changes: 1 addition & 1 deletion src/common/in_mem_overflow_buffer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace kuzu {
namespace common {

uint8_t* InMemOverflowBuffer::allocateSpace(uint64_t size) {
assert(size <= LARGE_PAGE_SIZE);
assert(size <= BufferPoolConstants::LARGE_PAGE_SIZE);
if (requireNewBlock(size)) {
allocateNewBlock();
}
Expand Down
2 changes: 1 addition & 1 deletion src/common/task_system/task_scheduler.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "common/task_system/task_scheduler.h"

#include "common/configs.h"
#include "common/constants.h"
#include "spdlog/spdlog.h"

using namespace kuzu::common;
Expand Down
2 changes: 1 addition & 1 deletion src/include/binder/expression/property_expression.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "expression.h"

namespace kuzu {
Expand Down
2 changes: 1 addition & 1 deletion src/include/catalog/catalog_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#include <unordered_map>
#include <unordered_set>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/types/types_include.h"

Expand Down
41 changes: 20 additions & 21 deletions src/include/common/configs.h → src/include/common/constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,7 @@ namespace common {
constexpr uint64_t DEFAULT_VECTOR_CAPACITY_LOG_2 = 11;
constexpr uint64_t DEFAULT_VECTOR_CAPACITY = (uint64_t)1 << DEFAULT_VECTOR_CAPACITY_LOG_2;

// Currently the system supports files with 2 different pages size, which we refer to as
// DEFAULT_PAGE_SIZE and LARGE_PAGE_SIZE. Default size of the page which is the unit of read/write
// to the database files, such as to store columns or lists. For now, this value cannot be changed.
// But technically it can change from 2^12 to 2^16. 2^12 lower bound is assuming the OS page size is
// 4K. 2^16 is because currently we leave 11 fixed number of bits for relOffInPage and the maximum
// number of bytes needed for an edge is 20 bytes so 11 + log_2(20) = 15.xxx, so certainly over
// 2^16-size pages, we cannot utilize the page for storing adjacency lists.
constexpr uint64_t DEFAULT_PAGE_SIZE_LOG_2 = 12;
constexpr uint64_t DEFAULT_PAGE_SIZE = 1 << DEFAULT_PAGE_SIZE_LOG_2;
// Page size for files with large pages, e.g., temporary files that are used by operators that may
// require large amounts of memory.
constexpr const uint64_t LARGE_PAGE_SIZE_LOG_2 = 18;
constexpr uint64_t LARGE_PAGE_SIZE = 1 << LARGE_PAGE_SIZE_LOG_2;

constexpr const double DEFAULT_HT_LOAD_FACTOR = 1.5;

constexpr const uint32_t VAR_LENGTH_EXTEND_MAX_DEPTH = 30;

// This is the default thread sleep time we use when a thread,
Expand All @@ -36,13 +21,28 @@ constexpr const uint64_t THREAD_SLEEP_TIME_WHEN_WAITING_IN_MICROS = 500;
// design is meant to not contain any page version arrays if a group of pages do not contain
// any updates to reduce our memory footprint.
constexpr const uint64_t MULTI_VERSION_FILE_PAGE_GROUP_SIZE = 64;

constexpr const uint64_t DEFAULT_CHECKPOINT_WAIT_TIMEOUT_FOR_TRANSACTIONS_TO_LEAVE_IN_MICROS =
5000000;

const std::string INTERNAL_ID_SUFFIX = "_id";

struct StorageConfig {
// Currently the system supports files with 2 different pages size, which we refer to as
// DEFAULT_PAGE_SIZE and LARGE_PAGE_SIZE. Default size of the page which is the unit of read/write
// to the database files, such as to store columns or lists. For now, this value cannot be changed.
// But technically it can change from 2^12 to 2^16. 2^12 lower bound is assuming the OS page size is
// 4K. 2^16 is because currently we leave 11 fixed number of bits for relOffInPage and the maximum
// number of bytes needed for an edge is 20 bytes so 11 + log_2(20) = 15.xxx, so certainly over
// 2^16-size pages, we cannot utilize the page for storing adjacency lists.
struct BufferPoolConstants {
static constexpr uint64_t DEFAULT_PAGE_SIZE_LOG_2 = 12;
static constexpr uint64_t DEFAULT_PAGE_SIZE = 1 << DEFAULT_PAGE_SIZE_LOG_2;
// Page size for files with large pages, e.g., temporary files that are used by operators that
// may require large amounts of memory.
static constexpr const uint64_t LARGE_PAGE_SIZE_LOG_2 = 18;
static constexpr uint64_t LARGE_PAGE_SIZE = 1 << LARGE_PAGE_SIZE_LOG_2;
};

struct StorageConstants {
// The default amount of memory pre-allocated to both the default and large pages buffer pool.
static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE = 1ull << 30; // (1GB)
static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE_FOR_TESTING = 1ull << 26; // (64MB)
Expand All @@ -67,7 +67,7 @@ struct StorageConfig {
constexpr static double ARRAY_RESIZING_FACTOR = 1.2;
};

struct ListsMetadataConfig {
struct ListsMetadataConstants {
// LIST_CHUNK_SIZE should strictly be a power of 2.
constexpr static uint16_t LISTS_CHUNK_SIZE_LOG_2 = 9;
constexpr static uint16_t LISTS_CHUNK_SIZE = 1 << LISTS_CHUNK_SIZE_LOG_2;
Expand All @@ -80,12 +80,12 @@ struct ListsMetadataConfig {
};

// Hash Index Configurations
struct HashIndexConfig {
struct HashIndexConstants {
static constexpr uint8_t SLOT_CAPACITY_LOG_2 = 2;
static constexpr uint8_t SLOT_CAPACITY = (uint64_t)1 << SLOT_CAPACITY_LOG_2;
};

struct CopyConfig {
struct CopyConstants {
// Size (in bytes) of the chunks to be read in Node/Rel Copier
static constexpr uint64_t CSV_READING_BLOCK_SIZE = 1 << 23;

Expand All @@ -108,7 +108,6 @@ struct CopyConfig {

struct EnumeratorKnobs {
static constexpr double PREDICATE_SELECTIVITY = 0.1;
static constexpr double RANDOM_LOOKUP_PENALTY = 1000;
static constexpr double FLAT_PROBE_PENALTY = 10;
};

Expand Down
14 changes: 7 additions & 7 deletions src/include/common/csv_reader/csv_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <fstream>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types_include.h"
#include "common/types/value.h"

Expand All @@ -15,12 +15,12 @@ namespace common {

struct CSVReaderConfig {
CSVReaderConfig()
: escapeChar{CopyConfig::DEFAULT_CSV_ESCAPE_CHAR},
delimiter{CopyConfig::DEFAULT_CSV_DELIMITER},
quoteChar{CopyConfig::DEFAULT_CSV_QUOTE_CHAR},
listBeginChar{CopyConfig::DEFAULT_CSV_LIST_BEGIN_CHAR},
listEndChar{CopyConfig::DEFAULT_CSV_LIST_END_CHAR},
hasHeader{CopyConfig::DEFAULT_CSV_HAS_HEADER} {}
: escapeChar{CopyConstants::DEFAULT_CSV_ESCAPE_CHAR},
delimiter{CopyConstants::DEFAULT_CSV_DELIMITER},
quoteChar{CopyConstants::DEFAULT_CSV_QUOTE_CHAR},
listBeginChar{CopyConstants::DEFAULT_CSV_LIST_BEGIN_CHAR},
listEndChar{CopyConstants::DEFAULT_CSV_LIST_END_CHAR},
hasHeader{CopyConstants::DEFAULT_CSV_HAS_HEADER} {}

char escapeChar;
char delimiter;
Expand Down
2 changes: 1 addition & 1 deletion src/include/common/data_chunk/sel_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <memory>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types.h"

namespace kuzu {
Expand Down
4 changes: 2 additions & 2 deletions src/include/common/in_mem_overflow_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ class InMemOverflowBuffer {

private:
inline bool requireNewBlock(uint64_t sizeToAllocate) {
if (sizeToAllocate > LARGE_PAGE_SIZE) {
if (sizeToAllocate > BufferPoolConstants::LARGE_PAGE_SIZE) {
throw RuntimeException("Require size " + std::to_string(sizeToAllocate) +
" greater than single block size " +
std::to_string(LARGE_PAGE_SIZE) + ".");
std::to_string(BufferPoolConstants::LARGE_PAGE_SIZE) + ".");
}
return currentBlock == nullptr ||
(currentBlock->currentOffset + sizeToAllocate) > currentBlock->size;
Expand Down
2 changes: 1 addition & 1 deletion src/include/common/null_mask.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <utility>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"

namespace kuzu {
namespace common {
Expand Down
2 changes: 1 addition & 1 deletion src/include/main/database.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <thread>

#include "common/api.h"
#include "common/configs.h"
#include "common/constants.h"
#include "kuzu_fwd.h"

namespace kuzu {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#include <string>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/utils.h"
#include "common/vector/value_vector.h"
Expand Down
2 changes: 1 addition & 1 deletion src/include/processor/operator/order_by/radix_sort.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <queue>

#include "common/configs.h"
#include "common/constants.h"
#include "common/vector/value_vector.h"
#include "processor/operator/order_by/key_block_merger.h"
#include "processor/operator/order_by/order_by_key_encoder.h"
Expand Down
6 changes: 4 additions & 2 deletions src/include/processor/result/factorized_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@ class DataBlock {

inline uint8_t* getData() const { return block->data; }
inline void resetNumTuplesAndFreeSize() {
freeSize = common::LARGE_PAGE_SIZE;
freeSize = common::BufferPoolConstants::LARGE_PAGE_SIZE;
numTuples = 0;
}
inline void resetToZero() { memset(block->data, 0, common::LARGE_PAGE_SIZE); }
inline void resetToZero() {
memset(block->data, 0, common::BufferPoolConstants::LARGE_PAGE_SIZE);
}

static void copyTuples(DataBlock* blockToCopyFrom, ft_tuple_idx_t tupleIdxToCopyFrom,
DataBlock* blockToCopyInto, ft_tuple_idx_t tupleIdxToCopyTo, uint32_t numTuplesToCopy,
Expand Down
5 changes: 3 additions & 2 deletions src/include/storage/buffer_manager/file_handle.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include <shared_mutex>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/file_utils.h"
#include "common/types/types.h"
Expand Down Expand Up @@ -101,7 +101,8 @@ class FileHandle {
void releasePageLock(common::page_idx_t pageIdx) { pageLocks[pageIdx]->clear(); }

inline uint64_t getPageSize() const {
return isLargePaged() ? common::LARGE_PAGE_SIZE : common::DEFAULT_PAGE_SIZE;
return isLargePaged() ? common::BufferPoolConstants::LARGE_PAGE_SIZE :
common::BufferPoolConstants::DEFAULT_PAGE_SIZE;
}

protected:
Expand Down
4 changes: 2 additions & 2 deletions src/include/storage/buffer_manager/memory_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include <mutex>
#include <stack>

#include "common/configs.h"
#include "common/constants.h"
#include "storage/buffer_manager/buffer_manager.h"

namespace kuzu {
Expand All @@ -15,7 +15,7 @@ struct MemoryBlock {

public:
explicit MemoryBlock(common::page_idx_t pageIdx, uint8_t* data)
: size(common::LARGE_PAGE_SIZE), pageIdx(pageIdx), data(data) {}
: size(common::BufferPoolConstants::LARGE_PAGE_SIZE), pageIdx(pageIdx), data(data) {}

public:
uint64_t size;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "file_handle.h"
#include "storage/buffer_manager/file_handle.h"
#include "storage/storage_utils.h"
Expand Down
2 changes: 1 addition & 1 deletion src/include/storage/index/hash_index_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ static constexpr uint64_t INDEX_HEADER_IDX_IN_ARRAY = 0;
*
* The slot data structure:
* Each slot (p/oSlot) consists of a slot header and several entries. The max number of entries in
* slot is given by HashIndexConfig::SLOT_CAPACITY. The size of the slot is given by
* slot is given by HashIndexConstants::SLOT_CAPACITY. The size of the slot is given by
* (sizeof(SlotHeader) + (SLOT_CAPACITY * sizeof(Entry)).
*
* SlotHeader: [numEntries, validityMask, nextOvfSlotId]
Expand Down
4 changes: 2 additions & 2 deletions src/include/storage/index/hash_index_slot.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <cstdint>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/internal_id_t.h"
#include "common/types/ku_string.h"

Expand Down Expand Up @@ -46,7 +46,7 @@ struct SlotEntry {
template<typename T>
struct Slot {
SlotHeader header;
SlotEntry<T> entries[common::HashIndexConfig::SLOT_CAPACITY];
SlotEntry<T> entries[common::HashIndexConstants::SLOT_CAPACITY];
};

} // namespace storage
Expand Down
13 changes: 8 additions & 5 deletions src/include/storage/storage_structure/disk_array.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types.h"
#include "storage/buffer_manager/versioned_file_handle.h"
#include "storage/wal/wal.h"
Expand All @@ -13,7 +13,8 @@ namespace storage {
class FileHandle;

static constexpr uint64_t NUM_PAGE_IDXS_PER_PIP =
(common::DEFAULT_PAGE_SIZE - sizeof(common::page_idx_t)) / sizeof(common::page_idx_t);
(common::BufferPoolConstants::DEFAULT_PAGE_SIZE - sizeof(common::page_idx_t)) /
sizeof(common::page_idx_t);

/**
* Header page of a disk array.
Expand Down Expand Up @@ -73,7 +74,7 @@ struct PIPUpdates {
* stable header page, i.e., the header page of the array is always in a pre-allocated page in the
* file. The header page contains the pointer to the first ``page indices page'' (pip). Each pip
* stores a list of page indices that store the ``array pages''. Each PIP also stores the pageIdx of
* the next PIP if one exists (or we use StorageConfig::NULL_PAGE_IDX as null). Array pages store
* the next PIP if one exists (or we use StorageConstants::NULL_PAGE_IDX as null). Array pages store
* the actual data in the array.
*
* Storage structures can use multiple disk arrays in a single file by giving each one a different
Expand Down Expand Up @@ -196,9 +197,11 @@ class BaseInMemDiskArray : public BaseDiskArray<U> {

protected:
inline uint64_t addInMemoryArrayPage(bool setToZero) {
inMemArrayPages.emplace_back(std::make_unique<uint8_t[]>(common::DEFAULT_PAGE_SIZE));
inMemArrayPages.emplace_back(
std::make_unique<uint8_t[]>(common::BufferPoolConstants::DEFAULT_PAGE_SIZE));
if (setToZero) {
memset(inMemArrayPages[inMemArrayPages.size() - 1].get(), 0, common::DEFAULT_PAGE_SIZE);
memset(inMemArrayPages[inMemArrayPages.size() - 1].get(), 0,
common::BufferPoolConstants::DEFAULT_PAGE_SIZE);
}
return inMemArrayPages.size() - 1;
}
Expand Down
Loading