Skip to content

Commit

Permalink
rename config to constants
Browse files Browse the repository at this point in the history
  • Loading branch information
ray6080 committed Feb 16, 2023
1 parent a879779 commit cec863d
Show file tree
Hide file tree
Showing 67 changed files with 288 additions and 265 deletions.
4 changes: 2 additions & 2 deletions src/binder/binder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ void Binder::validateTableExist(const Catalog& _catalog, std::string& tableName)
}

bool Binder::validateStringParsingOptionName(std::string& parsingOptionName) {
for (auto i = 0; i < std::size(CopyConfig::STRING_CSV_PARSING_OPTIONS); i++) {
if (parsingOptionName == CopyConfig::STRING_CSV_PARSING_OPTIONS[i]) {
for (auto i = 0; i < std::size(CopyConstants::STRING_CSV_PARSING_OPTIONS); i++) {
if (parsingOptionName == CopyConstants::STRING_CSV_PARSING_OPTIONS[i]) {
return true;
}
}
Expand Down
19 changes: 10 additions & 9 deletions src/common/csv_reader/csv_reader.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "common/csv_reader/csv_reader.h"

#include "common/configs.h"
#include "common/constants.h"
#include "common/type_utils.h"
#include "common/utils.h"
#include "spdlog/spdlog.h"
Expand All @@ -13,8 +13,8 @@ namespace common {

CSVReader::CSVReader(const std::string& fName, const CSVReaderConfig& config, uint64_t blockId)
: CSVReader{fName, config} {
readingBlockStartOffset = CopyConfig::CSV_READING_BLOCK_SIZE * blockId;
readingBlockEndOffset = CopyConfig::CSV_READING_BLOCK_SIZE * (blockId + 1);
readingBlockStartOffset = CopyConstants::CSV_READING_BLOCK_SIZE * blockId;
readingBlockEndOffset = CopyConstants::CSV_READING_BLOCK_SIZE * (blockId + 1);
auto isBeginningOfLine = false;
if (0 == readingBlockStartOffset) {
isBeginningOfLine = true;
Expand Down Expand Up @@ -233,10 +233,11 @@ uint8_t CSVReader::getBoolean() {
char* CSVReader::getString() {
setNextTokenIsProcessed();
auto strVal = line + linePtrStart;
if (strlen(strVal) > DEFAULT_PAGE_SIZE) {
logger->warn(StringUtils::getLongStringErrorMessage(strVal, DEFAULT_PAGE_SIZE));
if (strlen(strVal) > BufferPoolConstants::DEFAULT_PAGE_SIZE) {
logger->warn(
StringUtils::getLongStringErrorMessage(strVal, BufferPoolConstants::DEFAULT_PAGE_SIZE));
// If the std::string is too long, truncate it.
strVal[DEFAULT_PAGE_SIZE] = '\0';
strVal[BufferPoolConstants::DEFAULT_PAGE_SIZE] = '\0';
}
auto unicodeType = Utf8Proc::analyze(strVal, strlen(strVal));
if (unicodeType == UnicodeType::ASCII) {
Expand Down Expand Up @@ -307,10 +308,10 @@ std::unique_ptr<Value> CSVReader::getList(const DataType& dataType) {
}
}
auto numBytesOfOverflow = listVal.size() * Types::getDataTypeSize(dataType.typeID);
if (numBytesOfOverflow >= DEFAULT_PAGE_SIZE) {
if (numBytesOfOverflow >= BufferPoolConstants::DEFAULT_PAGE_SIZE) {
throw ReaderException(StringUtils::string_format(
"Maximum num bytes of a LIST is %d. Input list's num bytes is %d.", DEFAULT_PAGE_SIZE,
numBytesOfOverflow));
"Maximum num bytes of a LIST is %d. Input list's num bytes is %d.",
BufferPoolConstants::DEFAULT_PAGE_SIZE, numBytesOfOverflow));
}
return std::make_unique<Value>(
DataType(LIST, std::make_unique<DataType>(dataType)), std::move(listVal));
Expand Down
2 changes: 1 addition & 1 deletion src/common/task_system/task_scheduler.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "common/task_system/task_scheduler.h"

#include "common/configs.h"
#include "common/constants.h"
#include "spdlog/spdlog.h"

using namespace kuzu::common;
Expand Down
2 changes: 1 addition & 1 deletion src/include/binder/expression/property_expression.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "expression.h"

namespace kuzu {
Expand Down
2 changes: 1 addition & 1 deletion src/include/catalog/catalog_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#include <unordered_map>
#include <unordered_set>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/types/types_include.h"

Expand Down
41 changes: 20 additions & 21 deletions src/include/common/configs.h → src/include/common/constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,7 @@ namespace common {
constexpr uint64_t DEFAULT_VECTOR_CAPACITY_LOG_2 = 11;
constexpr uint64_t DEFAULT_VECTOR_CAPACITY = (uint64_t)1 << DEFAULT_VECTOR_CAPACITY_LOG_2;

// Currently the system supports files with 2 different pages size, which we refer to as
// DEFAULT_PAGE_SIZE and LARGE_PAGE_SIZE. Default size of the page which is the unit of read/write
// to the database files, such as to store columns or lists. For now, this value cannot be changed.
// But technically it can change from 2^12 to 2^16. 2^12 lower bound is assuming the OS page size is
// 4K. 2^16 is because currently we leave 11 fixed number of bits for relOffInPage and the maximum
// number of bytes needed for an edge is 20 bytes so 11 + log_2(20) = 15.xxx, so certainly over
// 2^16-size pages, we cannot utilize the page for storing adjacency lists.
constexpr uint64_t DEFAULT_PAGE_SIZE_LOG_2 = 12;
constexpr uint64_t DEFAULT_PAGE_SIZE = 1 << DEFAULT_PAGE_SIZE_LOG_2;
// Page size for files with large pages, e.g., temporary files that are used by operators that may
// require large amounts of memory.
constexpr const uint64_t LARGE_PAGE_SIZE_LOG_2 = 18;
constexpr uint64_t LARGE_PAGE_SIZE = 1 << LARGE_PAGE_SIZE_LOG_2;

constexpr const double DEFAULT_HT_LOAD_FACTOR = 1.5;

constexpr const uint32_t VAR_LENGTH_EXTEND_MAX_DEPTH = 30;

// This is the default thread sleep time we use when a thread,
Expand All @@ -36,13 +21,28 @@ constexpr const uint64_t THREAD_SLEEP_TIME_WHEN_WAITING_IN_MICROS = 500;
// design is meant to not contain any page version arrays if a group of pages do not contain
// any updates to reduce our memory footprint.
constexpr const uint64_t MULTI_VERSION_FILE_PAGE_GROUP_SIZE = 64;

constexpr const uint64_t DEFAULT_CHECKPOINT_WAIT_TIMEOUT_FOR_TRANSACTIONS_TO_LEAVE_IN_MICROS =
5000000;

const std::string INTERNAL_ID_SUFFIX = "_id";

struct StorageConfig {
// Currently the system supports files with 2 different pages size, which we refer to as
// DEFAULT_PAGE_SIZE and LARGE_PAGE_SIZE. Default size of the page which is the unit of read/write
// to the database files, such as to store columns or lists. For now, this value cannot be changed.
// But technically it can change from 2^12 to 2^16. 2^12 lower bound is assuming the OS page size is
// 4K. 2^16 is because currently we leave 11 fixed number of bits for relOffInPage and the maximum
// number of bytes needed for an edge is 20 bytes so 11 + log_2(20) = 15.xxx, so certainly over
// 2^16-size pages, we cannot utilize the page for storing adjacency lists.
struct BufferPoolConstants {
static constexpr uint64_t DEFAULT_PAGE_SIZE_LOG_2 = 12;
static constexpr uint64_t DEFAULT_PAGE_SIZE = 1 << DEFAULT_PAGE_SIZE_LOG_2;
// Page size for files with large pages, e.g., temporary files that are used by operators that
// may require large amounts of memory.
static constexpr const uint64_t LARGE_PAGE_SIZE_LOG_2 = 18;
static constexpr uint64_t LARGE_PAGE_SIZE = 1 << LARGE_PAGE_SIZE_LOG_2;
};

struct StorageConstants {
// The default amount of memory pre-allocated to both the default and large pages buffer pool.
static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE = 1ull << 30; // (1GB)
static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE_FOR_TESTING = 1ull << 26; // (64MB)
Expand All @@ -67,7 +67,7 @@ struct StorageConfig {
constexpr static double ARRAY_RESIZING_FACTOR = 1.2;
};

struct ListsMetadataConfig {
struct ListsMetadataConstants {
// LIST_CHUNK_SIZE should strictly be a power of 2.
constexpr static uint16_t LISTS_CHUNK_SIZE_LOG_2 = 9;
constexpr static uint16_t LISTS_CHUNK_SIZE = 1 << LISTS_CHUNK_SIZE_LOG_2;
Expand All @@ -80,12 +80,12 @@ struct ListsMetadataConfig {
};

// Hash Index Configurations
struct HashIndexConfig {
struct HashIndexConstants {
static constexpr uint8_t SLOT_CAPACITY_LOG_2 = 2;
static constexpr uint8_t SLOT_CAPACITY = (uint64_t)1 << SLOT_CAPACITY_LOG_2;
};

struct CopyConfig {
struct CopyConstants {
// Size (in bytes) of the chunks to be read in Node/Rel Copier
static constexpr uint64_t CSV_READING_BLOCK_SIZE = 1 << 23;

Expand All @@ -108,7 +108,6 @@ struct CopyConfig {

struct EnumeratorKnobs {
static constexpr double PREDICATE_SELECTIVITY = 0.1;
static constexpr double RANDOM_LOOKUP_PENALTY = 1000;
static constexpr double FLAT_PROBE_PENALTY = 10;
};

Expand Down
14 changes: 7 additions & 7 deletions src/include/common/csv_reader/csv_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <fstream>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types_include.h"
#include "common/types/value.h"

Expand All @@ -15,12 +15,12 @@ namespace common {

struct CSVReaderConfig {
CSVReaderConfig()
: escapeChar{CopyConfig::DEFAULT_CSV_ESCAPE_CHAR},
delimiter{CopyConfig::DEFAULT_CSV_DELIMITER},
quoteChar{CopyConfig::DEFAULT_CSV_QUOTE_CHAR},
listBeginChar{CopyConfig::DEFAULT_CSV_LIST_BEGIN_CHAR},
listEndChar{CopyConfig::DEFAULT_CSV_LIST_END_CHAR},
hasHeader{CopyConfig::DEFAULT_CSV_HAS_HEADER} {}
: escapeChar{CopyConstants::DEFAULT_CSV_ESCAPE_CHAR},
delimiter{CopyConstants::DEFAULT_CSV_DELIMITER},
quoteChar{CopyConstants::DEFAULT_CSV_QUOTE_CHAR},
listBeginChar{CopyConstants::DEFAULT_CSV_LIST_BEGIN_CHAR},
listEndChar{CopyConstants::DEFAULT_CSV_LIST_END_CHAR},
hasHeader{CopyConstants::DEFAULT_CSV_HAS_HEADER} {}

char escapeChar;
char delimiter;
Expand Down
2 changes: 1 addition & 1 deletion src/include/common/data_chunk/sel_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <memory>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types.h"

namespace kuzu {
Expand Down
4 changes: 2 additions & 2 deletions src/include/common/in_mem_overflow_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ class InMemOverflowBuffer {

private:
inline bool requireNewBlock(uint64_t sizeToAllocate) {
if (sizeToAllocate > LARGE_PAGE_SIZE) {
if (sizeToAllocate > BufferPoolConstants::LARGE_PAGE_SIZE) {
throw RuntimeException("Require size " + std::to_string(sizeToAllocate) +
" greater than single block size " +
std::to_string(LARGE_PAGE_SIZE) + ".");
std::to_string(BufferPoolConstants::LARGE_PAGE_SIZE) + ".");
}
return currentBlock == nullptr ||
(currentBlock->currentOffset + sizeToAllocate) > currentBlock->size;
Expand Down
2 changes: 1 addition & 1 deletion src/include/common/null_mask.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <utility>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"

namespace kuzu {
namespace common {
Expand Down
2 changes: 1 addition & 1 deletion src/include/main/database.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <thread>

#include "common/api.h"
#include "common/configs.h"
#include "common/constants.h"
#include "kuzu_fwd.h"

namespace kuzu {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#include <string>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/utils.h"
#include "common/vector/value_vector.h"
Expand Down
2 changes: 1 addition & 1 deletion src/include/processor/operator/order_by/radix_sort.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <queue>

#include "common/configs.h"
#include "common/constants.h"
#include "common/vector/value_vector.h"
#include "processor/operator/order_by/key_block_merger.h"
#include "processor/operator/order_by/order_by_key_encoder.h"
Expand Down
6 changes: 4 additions & 2 deletions src/include/processor/result/factorized_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@ class DataBlock {

inline uint8_t* getData() const { return block->data; }
inline void resetNumTuplesAndFreeSize() {
freeSize = common::LARGE_PAGE_SIZE;
freeSize = common::BufferPoolConstants::LARGE_PAGE_SIZE;
numTuples = 0;
}
inline void resetToZero() { memset(block->data, 0, common::LARGE_PAGE_SIZE); }
inline void resetToZero() {
memset(block->data, 0, common::BufferPoolConstants::LARGE_PAGE_SIZE);
}

static void copyTuples(DataBlock* blockToCopyFrom, ft_tuple_idx_t tupleIdxToCopyFrom,
DataBlock* blockToCopyInto, ft_tuple_idx_t tupleIdxToCopyTo, uint32_t numTuplesToCopy,
Expand Down
5 changes: 3 additions & 2 deletions src/include/storage/buffer_manager/file_handle.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include <shared_mutex>
#include <vector>

#include "common/configs.h"
#include "common/constants.h"
#include "common/exception.h"
#include "common/file_utils.h"
#include "common/types/types.h"
Expand Down Expand Up @@ -101,7 +101,8 @@ class FileHandle {
void releasePageLock(common::page_idx_t pageIdx) { pageLocks[pageIdx]->clear(); }

inline uint64_t getPageSize() const {
return isLargePaged() ? common::LARGE_PAGE_SIZE : common::DEFAULT_PAGE_SIZE;
return isLargePaged() ? common::BufferPoolConstants::LARGE_PAGE_SIZE :
common::BufferPoolConstants::DEFAULT_PAGE_SIZE;
}

protected:
Expand Down
4 changes: 2 additions & 2 deletions src/include/storage/buffer_manager/memory_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include <mutex>
#include <stack>

#include "common/configs.h"
#include "common/constants.h"
#include "storage/buffer_manager/buffer_manager.h"

namespace kuzu {
Expand All @@ -15,7 +15,7 @@ struct MemoryBlock {

public:
explicit MemoryBlock(common::page_idx_t pageIdx, uint8_t* data)
: size(common::LARGE_PAGE_SIZE), pageIdx(pageIdx), data(data) {}
: size(common::BufferPoolConstants::LARGE_PAGE_SIZE), pageIdx(pageIdx), data(data) {}

public:
uint64_t size;
Expand Down
2 changes: 1 addition & 1 deletion src/include/storage/buffer_manager/versioned_file_handle.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "file_handle.h"
#include "storage/buffer_manager/file_handle.h"
#include "storage/storage_utils.h"
Expand Down
2 changes: 1 addition & 1 deletion src/include/storage/index/hash_index_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ static constexpr uint64_t INDEX_HEADER_IDX_IN_ARRAY = 0;
*
* The slot data structure:
* Each slot (p/oSlot) consists of a slot header and several entries. The max number of entries in
* slot is given by HashIndexConfig::SLOT_CAPACITY. The size of the slot is given by
* slot is given by HashIndexConstants::SLOT_CAPACITY. The size of the slot is given by
* (sizeof(SlotHeader) + (SLOT_CAPACITY * sizeof(Entry)).
*
* SlotHeader: [numEntries, validityMask, nextOvfSlotId]
Expand Down
4 changes: 2 additions & 2 deletions src/include/storage/index/hash_index_slot.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <cstdint>

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/internal_id_t.h"
#include "common/types/ku_string.h"

Expand Down Expand Up @@ -46,7 +46,7 @@ struct SlotEntry {
template<typename T>
struct Slot {
SlotHeader header;
SlotEntry<T> entries[common::HashIndexConfig::SLOT_CAPACITY];
SlotEntry<T> entries[common::HashIndexConstants::SLOT_CAPACITY];
};

} // namespace storage
Expand Down
13 changes: 8 additions & 5 deletions src/include/storage/storage_structure/disk_array.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include "common/configs.h"
#include "common/constants.h"
#include "common/types/types.h"
#include "storage/buffer_manager/versioned_file_handle.h"
#include "storage/wal/wal.h"
Expand All @@ -13,7 +13,8 @@ namespace storage {
class FileHandle;

static constexpr uint64_t NUM_PAGE_IDXS_PER_PIP =
(common::DEFAULT_PAGE_SIZE - sizeof(common::page_idx_t)) / sizeof(common::page_idx_t);
(common::BufferPoolConstants::DEFAULT_PAGE_SIZE - sizeof(common::page_idx_t)) /
sizeof(common::page_idx_t);

/**
* Header page of a disk array.
Expand Down Expand Up @@ -73,7 +74,7 @@ struct PIPUpdates {
* stable header page, i.e., the header page of the array is always in a pre-allocated page in the
* file. The header page contains the pointer to the first ``page indices page'' (pip). Each pip
* stores a list of page indices that store the ``array pages''. Each PIP also stores the pageIdx of
* the next PIP if one exists (or we use StorageConfig::NULL_PAGE_IDX as null). Array pages store
* the next PIP if one exists (or we use StorageConstants::NULL_PAGE_IDX as null). Array pages store
* the actual data in the array.
*
* Storage structures can use multiple disk arrays in a single file by giving each one a different
Expand Down Expand Up @@ -196,9 +197,11 @@ class BaseInMemDiskArray : public BaseDiskArray<U> {

protected:
inline uint64_t addInMemoryArrayPage(bool setToZero) {
inMemArrayPages.emplace_back(std::make_unique<uint8_t[]>(common::DEFAULT_PAGE_SIZE));
inMemArrayPages.emplace_back(
std::make_unique<uint8_t[]>(common::BufferPoolConstants::DEFAULT_PAGE_SIZE));
if (setToZero) {
memset(inMemArrayPages[inMemArrayPages.size() - 1].get(), 0, common::DEFAULT_PAGE_SIZE);
memset(inMemArrayPages[inMemArrayPages.size() - 1].get(), 0,
common::BufferPoolConstants::DEFAULT_PAGE_SIZE);
}
return inMemArrayPages.size() - 1;
}
Expand Down
3 changes: 2 additions & 1 deletion src/include/storage/storage_structure/disk_overflow_file.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ class DiskOverflowFile : public StorageStructure {
constructOverflowStorageStructureIDAndFName(storageStructureIDAndFNameOfMainDBFile),
bufferManager, wal),
loggedNewOverflowFileNextBytePosRecord{false} {
nextBytePosToWriteTo = fileHandle.getNumPages() * common::DEFAULT_PAGE_SIZE;
nextBytePosToWriteTo =
fileHandle.getNumPages() * common::BufferPoolConstants::DEFAULT_PAGE_SIZE;
}

static inline StorageStructureIDAndFName constructOverflowStorageStructureIDAndFName(
Expand Down
Loading

0 comments on commit cec863d

Please sign in to comment.