Skip to content

Commit

Permalink
Merge pull request #1224 from kuzudb/remove-inmem-mode
Browse files Browse the repository at this point in the history 
remove in-memory-mode
  • Loading branch information
@acquamarin
acquamarin committed Feb 1, 2023
2 parents 497235d + d9246c1 commit c2838a6
Show file tree
Hide file tree
Showing 31 changed files with 112 additions and 162 deletions.
4 changes: 1 addition & 3 deletions src/include/main/database.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,9 @@ struct SystemConfig {
};

struct DatabaseConfig {
explicit DatabaseConfig(std::string databasePath, bool inMemoryMode = false)
: databasePath{std::move(databasePath)}, inMemoryMode{inMemoryMode} {}
explicit DatabaseConfig(std::string databasePath) : databasePath{std::move(databasePath)} {}

std::string databasePath;
bool inMemoryMode;
};

class Database {
Expand Down
2 changes: 1 addition & 1 deletion src/include/storage/storage_manager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class StorageManager {

public:
StorageManager(catalog::Catalog& catalog, BufferManager& bufferManager,
MemoryManager& memoryManager, bool isInMemoryMode, WAL* wal);
MemoryManager& memoryManager, WAL* wal);

~StorageManager() = default;

Expand Down
43 changes: 20 additions & 23 deletions src/include/storage/storage_structure/column.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ class Column : public BaseColumnOrList {

public:
Column(const StorageStructureIDAndFName& structureIDAndFName, const DataType& dataType,
size_t elementSize, BufferManager& bufferManager, bool isInMemory, WAL* wal)
size_t elementSize, BufferManager& bufferManager, WAL* wal)
: BaseColumnOrList{structureIDAndFName, dataType, elementSize, bufferManager,
true /*hasNULLBytes*/, isInMemory, wal} {};
true /*hasNULLBytes*/, wal} {};

Column(const StorageStructureIDAndFName& structureIDAndFName, const DataType& dataType,
BufferManager& bufferManager, bool isInMemory, WAL* wal)
BufferManager& bufferManager, WAL* wal)
: Column(structureIDAndFName, dataType, Types::getDataTypeSize(dataType), bufferManager,
isInMemory, wal){};
wal){};

void read(Transaction* transaction, const shared_ptr<ValueVector>& nodeIDVector,
const shared_ptr<ValueVector>& resultVector);
Expand Down Expand Up @@ -81,9 +81,9 @@ class Column : public BaseColumnOrList {
class PropertyColumnWithOverflow : public Column {
public:
PropertyColumnWithOverflow(const StorageStructureIDAndFName& structureIDAndFNameOfMainColumn,
const DataType& dataType, BufferManager& bufferManager, bool isInMemory, WAL* wal)
: Column{structureIDAndFNameOfMainColumn, dataType, bufferManager, isInMemory, wal},
diskOverflowFile{structureIDAndFNameOfMainColumn, bufferManager, isInMemory, wal} {}
const DataType& dataType, BufferManager& bufferManager, WAL* wal)
: Column{structureIDAndFNameOfMainColumn, dataType, bufferManager, wal},
diskOverflowFile{structureIDAndFNameOfMainColumn, bufferManager, wal} {}

inline void read(Transaction* transaction, const shared_ptr<ValueVector>& nodeIDVector,
const shared_ptr<ValueVector>& resultVector) {
Expand All @@ -104,9 +104,9 @@ class StringPropertyColumn : public PropertyColumnWithOverflow {

public:
StringPropertyColumn(const StorageStructureIDAndFName& structureIDAndFNameOfMainColumn,
const DataType& dataType, BufferManager& bufferManager, bool isInMemory, WAL* wal)
const DataType& dataType, BufferManager& bufferManager, WAL* wal)
: PropertyColumnWithOverflow{
structureIDAndFNameOfMainColumn, dataType, bufferManager, isInMemory, wal} {};
structureIDAndFNameOfMainColumn, dataType, bufferManager, wal} {};

void writeValueForSingleNodeIDPosition(offset_t nodeOffset,
const shared_ptr<ValueVector>& vectorToWriteFrom, uint32_t posInVectorToWriteFrom) override;
Expand Down Expand Up @@ -139,9 +139,9 @@ class ListPropertyColumn : public PropertyColumnWithOverflow {

public:
ListPropertyColumn(const StorageStructureIDAndFName& structureIDAndFNameOfMainColumn,
const DataType& dataType, BufferManager& bufferManager, bool isInMemory, WAL* wal)
const DataType& dataType, BufferManager& bufferManager, WAL* wal)
: PropertyColumnWithOverflow{
structureIDAndFNameOfMainColumn, dataType, bufferManager, isInMemory, wal} {};
structureIDAndFNameOfMainColumn, dataType, bufferManager, wal} {};

void writeValueForSingleNodeIDPosition(offset_t nodeOffset,
const shared_ptr<ValueVector>& vectorToWriteFrom, uint32_t posInVectorToWriteFrom) override;
Expand Down Expand Up @@ -173,9 +173,8 @@ class RelIDColumn : public Column {

public:
RelIDColumn(const StorageStructureIDAndFName& structureIDAndFName, BufferManager& bufferManager,
bool isInMemory, WAL* wal)
: Column{structureIDAndFName, DataType(INTERNAL_ID), sizeof(offset_t), bufferManager,
isInMemory, wal},
WAL* wal)
: Column{structureIDAndFName, DataType(INTERNAL_ID), sizeof(offset_t), bufferManager, wal},
commonTableID{structureIDAndFName.storageStructureID.columnFileID.relPropertyColumnID
.relNodeTableAndDir.relTableID} {
assert(structureIDAndFName.storageStructureID.columnFileID.columnType ==
Expand Down Expand Up @@ -217,10 +216,9 @@ class AdjColumn : public Column {

public:
AdjColumn(const StorageStructureIDAndFName& structureIDAndFName, BufferManager& bufferManager,
const NodeIDCompressionScheme& nodeIDCompressionScheme, bool isInMemory, WAL* wal)
const NodeIDCompressionScheme& nodeIDCompressionScheme, WAL* wal)
: Column{structureIDAndFName, DataType(INTERNAL_ID),
nodeIDCompressionScheme.getNumBytesForNodeIDAfterCompression(), bufferManager,
isInMemory, wal},
nodeIDCompressionScheme.getNumBytesForNodeIDAfterCompression(), bufferManager, wal},
nodeIDCompressionScheme(nodeIDCompressionScheme){};

private:
Expand Down Expand Up @@ -256,28 +254,27 @@ class ColumnFactory {

public:
static unique_ptr<Column> getColumn(const StorageStructureIDAndFName& structureIDAndFName,
const DataType& dataType, BufferManager& bufferManager, bool isInMemory, WAL* wal) {
const DataType& dataType, BufferManager& bufferManager, WAL* wal) {
switch (dataType.typeID) {
case INT64:
case DOUBLE:
case BOOL:
case DATE:
case TIMESTAMP:
case INTERVAL:
return make_unique<Column>(
structureIDAndFName, dataType, bufferManager, isInMemory, wal);
return make_unique<Column>(structureIDAndFName, dataType, bufferManager, wal);
case STRING:
return make_unique<StringPropertyColumn>(
structureIDAndFName, dataType, bufferManager, isInMemory, wal);
structureIDAndFName, dataType, bufferManager, wal);
case LIST:
return make_unique<ListPropertyColumn>(
structureIDAndFName, dataType, bufferManager, isInMemory, wal);
structureIDAndFName, dataType, bufferManager, wal);
case INTERNAL_ID:
assert(structureIDAndFName.storageStructureID.storageStructureType ==
StorageStructureType::COLUMN);
assert(structureIDAndFName.storageStructureID.columnFileID.columnType ==
ColumnType::REL_PROPERTY_COLUMN);
return make_unique<RelIDColumn>(structureIDAndFName, bufferManager, isInMemory, wal);
return make_unique<RelIDColumn>(structureIDAndFName, bufferManager, wal);
default:
throw StorageException("Invalid type for property column creation.");
}
Expand Down
10 changes: 2 additions & 8 deletions src/include/storage/storage_structure/disk_overflow_file.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,14 @@ class DiskOverflowFile : public StorageStructure {

public:
DiskOverflowFile(const StorageStructureIDAndFName& storageStructureIDAndFNameOfMainDBFile,
BufferManager& bufferManager, bool isInMemory, WAL* wal)
BufferManager& bufferManager, WAL* wal)
: StorageStructure(
constructOverflowStorageStructureIDAndFName(storageStructureIDAndFNameOfMainDBFile),
bufferManager, isInMemory, wal),
bufferManager, wal),
loggedNewOverflowFileNextBytePosRecord{false} {
nextBytePosToWriteTo = fileHandle.getNumPages() * DEFAULT_PAGE_SIZE;
}

~DiskOverflowFile() {
if (isInMemory_) {
StorageStructureUtils::unpinEachPageOfFile(fileHandle, bufferManager);
}
}

static inline StorageStructureIDAndFName constructOverflowStorageStructureIDAndFName(
const StorageStructureIDAndFName& storageStructureIDAndFNameForMainDBFile) {
StorageStructureIDAndFName copy = storageStructureIDAndFNameForMainDBFile;
Expand Down
50 changes: 24 additions & 26 deletions src/include/storage/storage_structure/lists/lists.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ class Lists : public BaseColumnOrList {
public:
Lists(const StorageStructureIDAndFName& storageStructureIDAndFName, const DataType& dataType,
const size_t& elementSize, shared_ptr<ListHeaders> headers, BufferManager& bufferManager,
bool isInMemory, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
WAL* wal, ListsUpdatesStore* listsUpdatesStore)
: Lists{storageStructureIDAndFName, dataType, elementSize, std::move(headers),
bufferManager, true /*hasNULLBytes*/, isInMemory, wal, listsUpdatesStore} {};
bufferManager, true /*hasNULLBytes*/, wal, listsUpdatesStore} {};
inline ListsMetadata& getListsMetadata() { return metadata; };
inline shared_ptr<ListHeaders> getHeaders() const { return headers; };
// TODO(Guodong): change the input to header.
Expand Down Expand Up @@ -109,9 +109,9 @@ class Lists : public BaseColumnOrList {
virtual inline NodeIDCompressionScheme* getNodeIDCompressionIfExists() { return nullptr; }
Lists(const StorageStructureIDAndFName& storageStructureIDAndFName, const DataType& dataType,
const size_t& elementSize, shared_ptr<ListHeaders> headers, BufferManager& bufferManager,
bool hasNULLBytes, bool isInMemory, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
bool hasNULLBytes, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
: BaseColumnOrList{storageStructureIDAndFName, dataType, elementSize, bufferManager,
hasNULLBytes, isInMemory, wal},
hasNULLBytes, wal},
storageStructureIDAndFName{storageStructureIDAndFName},
metadata{storageStructureIDAndFName, &bufferManager, wal}, headers{std::move(headers)},
listsUpdatesStore{listsUpdatesStore} {};
Expand All @@ -137,10 +137,10 @@ class PropertyListsWithOverflow : public Lists {
public:
PropertyListsWithOverflow(const StorageStructureIDAndFName& storageStructureIDAndFName,
const DataType& dataType, shared_ptr<ListHeaders> headers, BufferManager& bufferManager,
bool isInMemory, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
WAL* wal, ListsUpdatesStore* listsUpdatesStore)
: Lists{storageStructureIDAndFName, dataType, Types::getDataTypeSize(dataType),
std::move(headers), bufferManager, isInMemory, wal, listsUpdatesStore},
diskOverflowFile{storageStructureIDAndFName, bufferManager, isInMemory, wal} {}
std::move(headers), bufferManager, wal, listsUpdatesStore},
diskOverflowFile{storageStructureIDAndFName, bufferManager, wal} {}

private:
inline DiskOverflowFile* getDiskOverflowFileIfExists() override { return &diskOverflowFile; }
Expand All @@ -153,10 +153,10 @@ class StringPropertyLists : public PropertyListsWithOverflow {

public:
StringPropertyLists(const StorageStructureIDAndFName& storageStructureIDAndFName,
const shared_ptr<ListHeaders>& headers, BufferManager& bufferManager, bool isInMemory,
WAL* wal, ListsUpdatesStore* listsUpdatesStore)
const shared_ptr<ListHeaders>& headers, BufferManager& bufferManager, WAL* wal,
ListsUpdatesStore* listsUpdatesStore)
: PropertyListsWithOverflow{storageStructureIDAndFName, DataType{STRING}, headers,
bufferManager, isInMemory, wal, listsUpdatesStore} {};
bufferManager, wal, listsUpdatesStore} {};

private:
void readFromLargeList(
Expand All @@ -170,10 +170,9 @@ class ListPropertyLists : public PropertyListsWithOverflow {
public:
ListPropertyLists(const StorageStructureIDAndFName& storageStructureIDAndFName,
const DataType& dataType, const shared_ptr<ListHeaders>& headers,
BufferManager& bufferManager, bool isInMemory, WAL* wal,
ListsUpdatesStore* listsUpdatesStore)
BufferManager& bufferManager, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
: PropertyListsWithOverflow{storageStructureIDAndFName, dataType, headers, bufferManager,
isInMemory, wal, listsUpdatesStore} {};
wal, listsUpdatesStore} {};

private:
void readFromLargeList(
Expand All @@ -186,12 +185,12 @@ class AdjLists : public Lists {

public:
AdjLists(const StorageStructureIDAndFName& storageStructureIDAndFName,
BufferManager& bufferManager, NodeIDCompressionScheme nodeIDCompressionScheme,
bool isInMemory, WAL* wal, ListsUpdatesStore* listsUpdatesStore)
BufferManager& bufferManager, NodeIDCompressionScheme nodeIDCompressionScheme, WAL* wal,
ListsUpdatesStore* listsUpdatesStore)
: Lists{storageStructureIDAndFName, DataType(INTERNAL_ID),
nodeIDCompressionScheme.getNumBytesForNodeIDAfterCompression(),
make_shared<ListHeaders>(storageStructureIDAndFName, &bufferManager, wal),
bufferManager, false /* hasNullBytes */, isInMemory, wal, listsUpdatesStore},
bufferManager, false /* hasNullBytes */, wal, listsUpdatesStore},
nodeIDCompressionScheme{nodeIDCompressionScheme} {};

inline bool mayContainNulls() const override { return false; }
Expand Down Expand Up @@ -234,10 +233,10 @@ class RelIDList : public Lists {

public:
RelIDList(const StorageStructureIDAndFName& storageStructureIDAndFName,
shared_ptr<ListHeaders> headers, BufferManager& bufferManager, bool isInMemory, WAL* wal,
shared_ptr<ListHeaders> headers, BufferManager& bufferManager, WAL* wal,
ListsUpdatesStore* listsUpdatesStore)
: Lists{storageStructureIDAndFName, DataType{INTERNAL_ID}, sizeof(offset_t),
std::move(headers), bufferManager, isInMemory, wal, listsUpdatesStore} {}
std::move(headers), bufferManager, wal, listsUpdatesStore} {}

void setDeletedRelsIfNecessary(Transaction* transaction, ListHandle& listHandle,
const shared_ptr<ValueVector>& relIDVector) override;
Expand Down Expand Up @@ -266,8 +265,7 @@ class ListsFactory {
public:
static unique_ptr<Lists> getLists(const StorageStructureIDAndFName& structureIDAndFName,
const DataType& dataType, const shared_ptr<ListHeaders>& adjListsHeaders,
BufferManager& bufferManager, bool isInMemory, WAL* wal,
ListsUpdatesStore* listsUpdatesStore) {
BufferManager& bufferManager, WAL* wal, ListsUpdatesStore* listsUpdatesStore) {
assert(listsUpdatesStore != nullptr);
switch (dataType.typeID) {
case INT64:
Expand All @@ -277,17 +275,17 @@ class ListsFactory {
case TIMESTAMP:
case INTERVAL:
return make_unique<Lists>(structureIDAndFName, dataType,
Types::getDataTypeSize(dataType), adjListsHeaders, bufferManager, isInMemory, wal,
Types::getDataTypeSize(dataType), adjListsHeaders, bufferManager, wal,
listsUpdatesStore);
case STRING:
return make_unique<StringPropertyLists>(structureIDAndFName, adjListsHeaders,
bufferManager, isInMemory, wal, listsUpdatesStore);
return make_unique<StringPropertyLists>(
structureIDAndFName, adjListsHeaders, bufferManager, wal, listsUpdatesStore);
case LIST:
return make_unique<ListPropertyLists>(structureIDAndFName, dataType, adjListsHeaders,
bufferManager, isInMemory, wal, listsUpdatesStore);
bufferManager, wal, listsUpdatesStore);
case INTERNAL_ID:
return make_unique<RelIDList>(structureIDAndFName, adjListsHeaders, bufferManager,
isInMemory, wal, listsUpdatesStore);
return make_unique<RelIDList>(
structureIDAndFName, adjListsHeaders, bufferManager, wal, listsUpdatesStore);
default:
throw StorageException("Invalid type for property list creation.");
}
Expand Down
17 changes: 3 additions & 14 deletions src/include/storage/storage_structure/storage_structure.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,10 @@ class StorageStructure {

public:
StorageStructure(const StorageStructureIDAndFName& storageStructureIDAndFName,
BufferManager& bufferManager, bool isInMemory, WAL* wal)
BufferManager& bufferManager, WAL* wal)
: logger{LoggerUtils::getOrCreateLogger("storage")},
fileHandle{storageStructureIDAndFName, FileHandle::O_PERSISTENT_FILE_NO_CREATE},
bufferManager{bufferManager}, isInMemory_{isInMemory}, wal{wal} {
if (isInMemory) {
StorageStructureUtils::pinEachPageOfFile(fileHandle, bufferManager);
}
}
bufferManager{bufferManager}, wal{wal} {}

virtual ~StorageStructure() = default;

Expand All @@ -56,7 +52,6 @@ class StorageStructure {
shared_ptr<spdlog::logger> logger;
VersionedFileHandle fileHandle;
BufferManager& bufferManager;
bool isInMemory_;
WAL* wal;
};

Expand All @@ -67,12 +62,6 @@ class StorageStructure {
class BaseColumnOrList : public StorageStructure {

public:
~BaseColumnOrList() override {
if (isInMemory_) {
StorageStructureUtils::unpinEachPageOfFile(fileHandle, bufferManager);
}
}

// Maps the position of element in page to its byte offset in page.
// TODO(Everyone): we should slowly get rid of this function.
inline uint16_t mapElementPosToByteOffset(uint16_t pageElementPos) const {
Expand All @@ -90,7 +79,7 @@ class BaseColumnOrList : public StorageStructure {

BaseColumnOrList(const StorageStructureIDAndFName& storageStructureIDAndFName,
DataType dataType, const size_t& elementSize, BufferManager& bufferManager,
bool hasNULLBytes, bool isInMemory, WAL* wal);
bool hasNULLBytes, WAL* wal);

void readBySequentialCopy(Transaction* transaction, const shared_ptr<ValueVector>& vector,
PageElementCursor& cursor,
Expand Down
5 changes: 2 additions & 3 deletions src/include/storage/store/node_table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class NodeTable {

public:
NodeTable(NodesStatisticsAndDeletedIDs* nodesStatisticsAndDeletedIDs,
BufferManager& bufferManager, bool isInMemory, WAL* wal, NodeTableSchema* nodeTableSchema);
BufferManager& bufferManager, WAL* wal, NodeTableSchema* nodeTableSchema);

void initializeData(NodeTableSchema* nodeTableSchema);

Expand Down Expand Up @@ -45,7 +45,7 @@ class NodeTable {
propertyColumns.emplace(property.propertyID,
ColumnFactory::getColumn(StorageUtils::getNodePropertyColumnStructureIDAndFName(
wal->getDirectory(), property),
property.dataType, bufferManager, isInMemory, wal));
property.dataType, bufferManager, wal));
}

offset_t addNodeAndResetProperties(ValueVector* primaryKeyVector);
Expand All @@ -61,7 +61,6 @@ class NodeTable {
unordered_map<property_id_t, unique_ptr<Column>> propertyColumns;
unique_ptr<PrimaryKeyIndex> pkIndex;
table_id_t tableID;
bool isInMemory;
BufferManager& bufferManager;
WAL* wal;
};
Expand Down
6 changes: 2 additions & 4 deletions src/include/storage/store/nodes_store.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace storage {
class NodesStore {

public:
NodesStore(const Catalog& catalog, BufferManager& bufferManager, bool isInMemoryMode, WAL* wal);
NodesStore(const Catalog& catalog, BufferManager& bufferManager, WAL* wal);

inline Column* getNodePropertyColumn(table_id_t tableID, uint64_t propertyIdx) const {
return nodeTables.at(tableID)->getPropertyColumn(propertyIdx);
Expand All @@ -34,7 +34,7 @@ class NodesStore {
inline void createNodeTable(
table_id_t tableID, BufferManager* bufferManager, WAL* wal, Catalog* catalog) {
nodeTables[tableID] = make_unique<NodeTable>(&nodesStatisticsAndDeletedIDs, *bufferManager,
isInMemoryMode, wal, catalog->getReadOnlyVersion()->getNodeTableSchema(tableID));
wal, catalog->getReadOnlyVersion()->getNodeTableSchema(tableID));
}
inline void removeNodeTable(table_id_t tableID) {
nodeTables.erase(tableID);
Expand All @@ -49,8 +49,6 @@ class NodesStore {
private:
unordered_map<table_id_t, unique_ptr<NodeTable>> nodeTables;
NodesStatisticsAndDeletedIDs nodesStatisticsAndDeletedIDs;
// Used to dynamically create nodeTables during checkpointing.
bool isInMemoryMode;
};

} // namespace storage
Expand Down
Loading

0 comments on commit c2838a6

Please sign in to comment.