diff --git a/src/include/common/utils.h b/src/include/common/utils.h index 4010b63218b..f41f0e517dd 100644 --- a/src/include/common/utils.h +++ b/src/include/common/utils.h @@ -10,7 +10,6 @@ #include #include "common/constants.h" -#include "common/types/types.h" #include "exception.h" #include "spdlog/fmt/fmt.h" diff --git a/src/include/storage/buffer_manager/buffer_pool.h b/src/include/storage/buffer_manager/buffer_pool.h deleted file mode 100644 index de03738afea..00000000000 --- a/src/include/storage/buffer_manager/buffer_pool.h +++ /dev/null @@ -1,141 +0,0 @@ -#pragma once - -#include -#include - -#include "common/metric.h" -#include "storage/buffer_manager/buffer_managed_file_handle.h" - -namespace spdlog { -class logger; -} - -namespace kuzu { -namespace storage { - -struct BufferManagerMetrics { - uint64_t numPins{0}; - // Number of pinning operations that required eviction from a Frame. - uint64_t numEvicts{0}; - // Number of failed tries to evict the page from a Frame. This is incremented if either the - // eviction routine fails to get the lock on the page that is in the Frame or the pinCount of - // the Frame has increased after taking the locks of Frame and page. - uint64_t numEvictFails{0}; - // Number of failed tried to evict the page frame a Frame because the Frame has been recently - // accessed and hence is given a second chance. - uint64_t numRecentlyAccessedWalkover{0}; - uint64_t numCacheHit{0}; - uint64_t numCacheMiss{0}; - uint64_t numDirtyPageWriteIO{0}; -}; - -// A frame is a unit of buffer space having a fixed size of 4KB, where a single file page is -// read from the disk. Frame also stores other metadata to locate and maintain this buffer in the -// Buffer Manager. -class Frame { - friend class BufferPool; - -public: - explicit Frame(common::page_offset_t pageSize, uint8_t* buffer); - ~Frame() noexcept(false); - -private: - void resetFrameWithoutLock(); - bool acquireFrameLock(bool block); - void releaseFrameLock() { frameLock.clear(); } - void setIsDirty(bool _isDirty) { isDirty = _isDirty; } - void releaseBuffer(); - -private: - // fileHandlePtr and pageIdx identify the file and the page in file whose data the buffer is - // maintaining. pageIdx of -1u means that the frame is empty, i.e. it has no data. - std::atomic fileHandlePtr; - std::atomic pageIdx; - std::atomic pinCount; - - bool recentlyAccessed; - bool isDirty; - uint8_t* buffer; - common::page_offset_t pageSize; - std::atomic_flag frameLock; -}; - -// The BufferPool is a cache of file pages of a fixed size. It provides the high-level functionality -// of pin() and unpin() pages of files in memory and operates via their FileHandles -// to make the page data available in one of the frames. It uses CLOCK replacement policy to evict -// pages from frames, which is an approximate LRU policy that is based of FIFO-like operations. -class BufferPool { - friend class BufferManager; - -public: - BufferPool(uint64_t pageSize, uint64_t maxSize); - - uint8_t* pin(BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - - // Pins a new page that has been added to the file. This means that the BufferManager does not - // need to read the page from the file for now. Ensuring that the given pageIdx is new is the - // responsibility of the caller. - uint8_t* pinWithoutReadingFromFile( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - - uint8_t* pinWithoutAcquiringPageLock( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx, bool doNotReadFromFile); - - void setPinnedPageDirty(BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - - // The function assumes that the requested page is already pinned. - void unpin(BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - - void unpinWithoutAcquiringPageLock( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - - // Note: These two functions that remove pages from frames is not designed for concurrency and - // therefore not tested under concurrency. If this is called while other threads are accessing - // the BM, it should work safely but this is not tested. - void removeFilePagesFromFrames(BufferManagedFileHandle& fileHandle); - - void flushAllDirtyPagesInFrames(BufferManagedFileHandle& fileHandle); - void updateFrameIfPageIsInFrameWithoutPageOrFrameLock( - BufferManagedFileHandle& fileHandle, uint8_t* newPage, common::page_idx_t pageIdx); - - void removePageFromFrameWithoutFlushingIfNecessary( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx); - -private: - uint8_t* pin( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx, bool doNotReadFromFile); - - common::page_idx_t claimAFrame( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx, bool doNotReadFromFile); - - bool fillEmptyFrame(common::page_idx_t frameIdx, BufferManagedFileHandle& fileHandle, - common::page_idx_t pageIdx, bool doNotReadFromFile); - - bool tryEvict(common::page_idx_t frameIdx, BufferManagedFileHandle& fileHandle, - common::page_idx_t pageIdx, bool doNotReadFromFile); - - void moveClockHand(uint64_t newClockHand); - // Performs 2 actions: - // 1) Clears the contents of the frame. - // 2) Unswizzles the pageIdx in the frame. - void clearFrameAndUnswizzleWithoutLock(const std::unique_ptr& frame, - BufferManagedFileHandle& fileHandleInFrame, common::page_idx_t pageIdxInFrame); - void readNewPageIntoFrame(Frame& frame, BufferManagedFileHandle& fileHandle, - common::page_idx_t pageIdx, bool doNotReadFromFile); - - void flushIfDirty(const std::unique_ptr& frame); - - void removePageFromFrame( - BufferManagedFileHandle& fileHandle, common::page_idx_t pageIdx, bool shouldFlush); - -private: - std::shared_ptr logger; - uint64_t pageSize; - std::vector> bufferCache; - std::atomic clockHand; - common::page_idx_t numFrames; - BufferManagerMetrics bmMetrics; -}; - -} // namespace storage -} // namespace kuzu diff --git a/src/storage/buffer_manager/bm_managed_file_handle.cpp b/src/storage/buffer_manager/bm_managed_file_handle.cpp index fb909cd1414..55e7f3c62fa 100644 --- a/src/storage/buffer_manager/bm_managed_file_handle.cpp +++ b/src/storage/buffer_manager/bm_managed_file_handle.cpp @@ -155,6 +155,7 @@ void BMFileHandle::clearWALPageVersionIfNecessary(common::page_idx_t pageIdx) { } createPageVersionGroupIfNecessary(pageIdx); setWALPageVersionNoLock(pageIdx, UINT32_MAX); + // TODO(Guodong): Why do we release lock here? Need to understand how the lock was acquired. releasePageLock(pageIdx); } diff --git a/src/storage/buffer_manager/buffer_manager.cpp b/src/storage/buffer_manager/buffer_manager.cpp index f4c406a42af..1aa514b188e 100644 --- a/src/storage/buffer_manager/buffer_manager.cpp +++ b/src/storage/buffer_manager/buffer_manager.cpp @@ -257,7 +257,6 @@ void BufferManager::removePageFromFrame( fileHandle.acquirePageLock(pageIdx, LockMode::SPIN); auto pageState = fileHandle.getPageState(pageIdx); if (pageState && pageState->isCached()) { - assert(pageState->getPinCount() == 0); if (shouldFlush) { flushIfDirtyWithoutLock(fileHandle, pageIdx); } diff --git a/src/storage/buffer_manager/buffer_pool.cpp b/src/storage/buffer_manager/buffer_pool.cpp deleted file mode 100644 index 6f8c6d23c6b..00000000000 --- a/src/storage/buffer_manager/buffer_pool.cpp +++ /dev/null @@ -1,292 +0,0 @@ -#include "storage/buffer_manager/buffer_pool.h" - -#include - -#include "common/constants.h" -#include "common/exception.h" -#include "common/utils.h" -#include "spdlog/spdlog.h" - -using namespace kuzu::common; - -namespace kuzu { -namespace storage { - -Frame::Frame(page_offset_t pageSize, std::uint8_t* buffer) - : frameLock{ATOMIC_FLAG_INIT}, pageSize{pageSize}, buffer{buffer} { - resetFrameWithoutLock(); -} - -Frame::~Frame() noexcept(false) { - auto count = pinCount.load(); - if (0 != count && -1u != count) { - throw BufferManagerException( - "Deleting buffer that is still pinned. pinCount: " + std::to_string(count) + - " pageIdx: " + std::to_string(pageIdx)); - } -} - -void Frame::resetFrameWithoutLock() { - fileHandlePtr = -1u; - pageIdx = -1u; - pinCount = -1u; - recentlyAccessed = false; - isDirty = false; -} - -bool Frame::acquireFrameLock(bool block) { - if (block) { - while (frameLock.test_and_set()) // spinning - ; - return true; - } - return !frameLock.test_and_set(); -} - -void Frame::releaseBuffer() { - int error = madvise(buffer, pageSize, MADV_DONTNEED); - if (error) { - throw BufferManagerException("Releasing frame buffer failed with error code " + - std::to_string(error) + ": " + - std::string(std::strerror(errno))); - } -} - -BufferPool::BufferPool(uint64_t pageSize, uint64_t maxSize) - : logger{LoggerUtils::getLogger(LoggerConstants::LoggerEnum::BUFFER_MANAGER)}, - pageSize{pageSize}, clockHand{0}, - numFrames((page_idx_t)(ceil((double)maxSize / (double)pageSize))) { - assert(pageSize == BufferPoolConstants::DEFAULT_PAGE_SIZE || - pageSize == BufferPoolConstants::LARGE_PAGE_SIZE); - auto mmapRegion = (uint8_t*)mmap( - NULL, (numFrames * pageSize), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - for (auto i = 0u; i < numFrames; ++i) { - auto buffer = mmapRegion + (i * pageSize); - bufferCache.emplace_back(std::make_unique(pageSize, buffer)); - } - logger->info("Initialize buffer pool with the max size {}B, #{}byte-pages {}.", maxSize, - pageSize, numFrames); -} - -uint8_t* BufferPool::pin(BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - return pin(fileHandle, pageIdx, false /* read page from file */); -} - -uint8_t* BufferPool::pinWithoutReadingFromFile( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - return pin(fileHandle, pageIdx, true /* do not read page from file */); -} - -void BufferPool::removeFilePagesFromFrames(BufferManagedFileHandle& fileHandle) { - for (auto pageIdx = 0u; pageIdx < fileHandle.getNumPages(); ++pageIdx) { - removePageFromFrame(fileHandle, pageIdx, false /* do not flush */); - } -} - -void BufferPool::removePageFromFrame( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx, bool shouldFlush) { - fileHandle.acquirePageLock(pageIdx, true /*block*/); - auto frameIdx = fileHandle.getFrameIdx(pageIdx); - if (BufferManagedFileHandle::isAFrame(frameIdx)) { - auto& frame = bufferCache[frameIdx]; - frame->acquireFrameLock(true /* block */); - if (shouldFlush) { - flushIfDirty(frame); - } - clearFrameAndUnswizzleWithoutLock(frame, fileHandle, pageIdx); - frame->releaseBuffer(); - frame->releaseFrameLock(); - } - fileHandle.releasePageLock(pageIdx); -} - -void BufferPool::removePageFromFrameWithoutFlushingIfNecessary( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - if (pageIdx >= fileHandle.getNumPages()) { - return; - } - removePageFromFrame(fileHandle, pageIdx, false /* do not flush */); -} - -void BufferPool::flushAllDirtyPagesInFrames(BufferManagedFileHandle& fileHandle) { - for (auto pageIdx = 0u; pageIdx < fileHandle.getNumPages(); ++pageIdx) { - removePageFromFrame(fileHandle, pageIdx, true /* flush */); - } -} - -void BufferPool::updateFrameIfPageIsInFrameWithoutPageOrFrameLock( - BufferManagedFileHandle& fileHandle, uint8_t* newPage, page_idx_t pageIdx) { - auto frameIdx = fileHandle.getFrameIdx(pageIdx); - if (BufferManagedFileHandle::isAFrame(frameIdx)) { - memcpy(bufferCache[frameIdx]->buffer, newPage, BufferPoolConstants::DEFAULT_PAGE_SIZE); - } -} - -uint8_t* BufferPool::pin( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx, bool doNotReadFromFile) { - fileHandle.acquirePageLock(pageIdx, true /*block*/); - auto retVal = pinWithoutAcquiringPageLock(fileHandle, pageIdx, doNotReadFromFile); - fileHandle.releasePageLock(pageIdx); - return retVal; -} - -uint8_t* BufferPool::pinWithoutAcquiringPageLock( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx, bool doNotReadFromFile) { - auto frameIdx = fileHandle.getFrameIdx(pageIdx); - if (BufferManagedFileHandle::isAFrame(frameIdx)) { - auto& frame = bufferCache[frameIdx]; - frame->pinCount.fetch_add(1); - frame->recentlyAccessed = true; - bmMetrics.numCacheHit += 1; - } else { - frameIdx = claimAFrame(fileHandle, pageIdx, doNotReadFromFile); - fileHandle.swizzle(pageIdx, frameIdx); - if (!doNotReadFromFile) { - bmMetrics.numCacheMiss += 1; - } - } - bmMetrics.numPins += 1; - return bufferCache[fileHandle.getFrameIdx(pageIdx)]->buffer; -} - -void BufferPool::setPinnedPageDirty(BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - fileHandle.acquirePageLock(pageIdx, true /*block*/); - auto frameIdx = fileHandle.getFrameIdx(pageIdx); - if (!BufferManagedFileHandle::isAFrame((frameIdx)) || - (bufferCache[frameIdx]->pinCount.load() < 1)) { - fileHandle.releasePageLock(pageIdx); - throw BufferManagerException("If a page is not in memory or is not pinned, cannot set " - "it to isDirty = true.filePath: " + - fileHandle.getFileInfo()->path + - " pageIdx: " + std::to_string(pageIdx) + "."); - } - bufferCache[frameIdx]->setIsDirty(true /* isDirty */); - fileHandle.releasePageLock(pageIdx); -} - -page_idx_t BufferPool::claimAFrame( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx, bool doNotReadFromFile) { - auto localClockHand = clockHand.load(); - auto startFrame = localClockHand % numFrames; - for (auto i = 0u; i < 2 * numFrames; ++i) { - auto frameIdx = (startFrame + i) % numFrames; - auto pinCount = bufferCache[frameIdx]->pinCount.load(); - if ((-1u == pinCount && fillEmptyFrame(frameIdx, fileHandle, pageIdx, doNotReadFromFile)) || - (0u == pinCount && tryEvict(frameIdx, fileHandle, pageIdx, doNotReadFromFile))) { - moveClockHand(localClockHand + i + 1); - return frameIdx; - } - } - throw BufferManagerException("Cannot find a frame to evict from."); -} - -bool BufferPool::fillEmptyFrame(page_idx_t frameIdx, BufferManagedFileHandle& fileHandle, - page_idx_t pageIdx, bool doNotReadFromFile) { - auto& frame = bufferCache[frameIdx]; - if (!frame->acquireFrameLock(false)) { - return false; - } - if (-1u == frame->pinCount.load()) { - readNewPageIntoFrame(*frame, fileHandle, pageIdx, doNotReadFromFile); - frame->releaseFrameLock(); - return true; - } - frame->releaseFrameLock(); - return false; -} - -bool BufferPool::tryEvict(page_idx_t frameIdx, BufferManagedFileHandle& fileHandle, - page_idx_t pageIdx, bool doNotReadFromFile) { - auto& frame = bufferCache[frameIdx]; - if (frame->recentlyAccessed) { - frame->recentlyAccessed = false; - bmMetrics.numRecentlyAccessedWalkover += 1; - return false; - } - if (!frame->acquireFrameLock(false)) { - return false; - } - auto pageIdxInFrame = frame->pageIdx.load(); - auto fileHandleInFrame = - reinterpret_cast(frame->fileHandlePtr.load()); - if (!fileHandleInFrame->acquirePageLock(pageIdxInFrame, false)) { - bmMetrics.numEvictFails += 1; - frame->releaseFrameLock(); - return false; - } - // We check pinCount again after acquiring the lock on page currently residing in the frame. At - // this point in time, no other thread can change the pinCount. - if (0u != frame->pinCount.load()) { - bmMetrics.numEvictFails += 1; - fileHandleInFrame->releasePageLock(pageIdxInFrame); - frame->releaseFrameLock(); - return false; - } - // Else, flush out the frame into the file page if the frame is dirty. Then remove the page from - // the frame and release the lock on it. - flushIfDirty(frame); - clearFrameAndUnswizzleWithoutLock(frame, *fileHandleInFrame, pageIdxInFrame); - fileHandleInFrame->releasePageLock(pageIdxInFrame); - // Update the frame information and release the lock on frame. - readNewPageIntoFrame(*frame, fileHandle, pageIdx, doNotReadFromFile); - frame->releaseFrameLock(); - bmMetrics.numEvicts += 1; - return true; -} - -void BufferPool::flushIfDirty(const std::unique_ptr& frame) { - auto fileHandleInFrame = reinterpret_cast(frame->fileHandlePtr.load()); - auto pageIdxInFrame = frame->pageIdx.load(); - if (frame->isDirty) { - bmMetrics.numDirtyPageWriteIO += 1; - fileHandleInFrame->writePage(frame->buffer, pageIdxInFrame); - } -} - -void BufferPool::clearFrameAndUnswizzleWithoutLock(const std::unique_ptr& frame, - BufferManagedFileHandle& fileHandleInFrame, page_idx_t pageIdxInFrame) { - frame->resetFrameWithoutLock(); - fileHandleInFrame.unswizzle(pageIdxInFrame); -} - -void BufferPool::readNewPageIntoFrame( - Frame& frame, BufferManagedFileHandle& fileHandle, page_idx_t pageIdx, bool doNotReadFromFile) { - frame.pinCount.store(1); - frame.recentlyAccessed = true; - frame.isDirty = false; - frame.pageIdx.store(pageIdx); - frame.fileHandlePtr.store(reinterpret_cast(&fileHandle)); - if (!doNotReadFromFile) { - fileHandle.readPage(frame.buffer, pageIdx); - } -} - -void BufferPool::moveClockHand(uint64_t newClockHand) { - do { - auto currClockHand = clockHand.load(); - if (currClockHand > newClockHand) { - return; - } - if (clockHand.compare_exchange_strong( - currClockHand, newClockHand, std::memory_order_seq_cst)) { - return; - } - } while (true); -} - -void BufferPool::unpin(BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - fileHandle.acquirePageLock(pageIdx, true /*block*/); - unpinWithoutAcquiringPageLock(fileHandle, pageIdx); - fileHandle.releasePageLock(pageIdx); -} - -void BufferPool::unpinWithoutAcquiringPageLock( - BufferManagedFileHandle& fileHandle, page_idx_t pageIdx) { - auto& frame = bufferCache[fileHandle.getFrameIdx(pageIdx)]; - // `count` is the value of `pinCount` before sub. - auto count = frame->pinCount.fetch_sub(1); - assert(count >= 1); -} - -} // namespace storage -} // namespace kuzu