From c2372d3119568177262055cdefefb30b08956b10 Mon Sep 17 00:00:00 2001 From: udi-speedb <106253580+udi-speedb@users.noreply.github.com> Date: Tue, 14 Feb 2023 19:55:51 +0200 Subject: [PATCH] Report to Log the Index size per CF - LRU Cache Only (#338) --- cache/cache.cc | 51 +++ cache/cache_bench_tool.cc | 3 +- cache/cache_entry_roles.cc | 15 + cache/cache_test.cc | 6 +- cache/charged_cache.cc | 12 +- cache/charged_cache.h | 14 +- cache/clock_cache.cc | 10 +- cache/clock_cache.h | 12 +- cache/fast_lru_cache.cc | 8 +- cache/fast_lru_cache.h | 12 +- cache/lru_cache.cc | 11 +- cache/lru_cache.h | 22 +- cache/lru_cache_test.cc | 67 ++-- cache/sharded_cache.cc | 15 +- cache/sharded_cache.h | 25 +- db/column_family.cc | 11 + db/column_family.h | 5 + db/db_basic_test.cc | 12 +- db/db_block_cache_test.cc | 333 +++++++++++++++++- db/db_test_util.cc | 5 +- db/db_test_util.h | 24 +- db/internal_stats.cc | 101 +++++- db/internal_stats.h | 24 +- db/table_cache.cc | 18 +- db/table_cache.h | 5 + include/rocksdb/cache.h | 60 +++- include/rocksdb/db.h | 4 + .../speedb_db_bloom_filter_test.cc | 12 +- .../block_based/block_based_table_builder.cc | 2 +- .../block_based/block_based_table_factory.cc | 2 +- table/block_based/block_based_table_reader.cc | 14 +- table/block_based/block_based_table_reader.h | 10 +- .../block_based_table_reader_impl.h | 6 +- table/table_builder.h | 2 + utilities/cache_dump_load_impl.cc | 4 +- utilities/cache_dump_load_impl.h | 2 +- utilities/simulator_cache/sim_cache.cc | 10 +- 37 files changed, 787 insertions(+), 162 deletions(-) diff --git a/cache/cache.cc b/cache/cache.cc index 7d23fb7579..1cc8ecec48 100644 --- a/cache/cache.cc +++ b/cache/cache.cc @@ -126,4 +126,55 @@ Status Cache::CreateFromString(const ConfigOptions& config_options, } return status; } + +// ================================================================================================================================== +Cache::ItemOwnerId Cache::ItemOwnerIdAllocator::Allocate() { + // In practice, onwer-ids are allocated and freed when cf-s + // are created and destroyed => relatively rare => paying + // the price to always lock the mutex and simplify the code + std::lock_guard lock(free_ids_mutex_); + + // First allocate from the free list if possible + if (free_ids_.empty() == false) { + auto allocated_id = free_ids_.front(); + free_ids_.pop_front(); + return allocated_id; + } + + // Nothing on the free list - try to allocate from the + // next item counter if not yet exhausted + if (has_wrapped_around_) { + // counter exhausted, allocation not possible + return kUnknownItemId; + } + + auto allocated_id = next_item_owner_id_++; + + if (allocated_id == kMaxOwnerItemId) { + has_wrapped_around_ = true; + } + + return allocated_id; +} + +void Cache::ItemOwnerIdAllocator::Free(ItemOwnerId* id) { + if (*id != kUnknownItemId) { + std::lock_guard lock(free_ids_mutex_); + // The freed id is lost but this is a luxury feature. We can't + // pay too much space to support it. + if (free_ids_.size() < kMaxFreeItemOwnersIdListSize) { + free_ids_.push_back(*id); + } + *id = kUnknownItemId; + } +} + +Cache::ItemOwnerId Cache::GetNextItemOwnerId() { + return owner_id_allocator_.Allocate(); +} + +void Cache::DiscardItemOwnerId(ItemOwnerId* item_owner_id) { + owner_id_allocator_.Free(item_owner_id); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/cache/cache_bench_tool.cc b/cache/cache_bench_tool.cc index e80aad8fe2..1a6bac1725 100644 --- a/cache/cache_bench_tool.cc +++ b/cache/cache_bench_tool.cc @@ -485,7 +485,8 @@ class CacheBench { total_entry_count = 0; deleters.clear(); auto fn = [&](const Slice& key, void* /*value*/, size_t charge, - Cache::DeleterFn deleter) { + Cache::DeleterFn deleter, + Cache::ItemOwnerId /* item_owner_id */) { total_key_size += key.size(); total_charge += charge; ++total_entry_count; diff --git a/cache/cache_entry_roles.cc b/cache/cache_entry_roles.cc index b27349554d..1632a8d63d 100644 --- a/cache/cache_entry_roles.cc +++ b/cache/cache_entry_roles.cc @@ -101,6 +101,21 @@ std::string BlockCacheEntryStatsMapKeys::UsedPercent(CacheEntryRole role) { return GetPrefixedCacheEntryRoleName(kPrefix, role); } +const std::string& BlockCacheCfStatsMapKeys::CfName() { + static const std::string kCfName = "cf_name"; + return kCfName; +} + +const std::string& BlockCacheCfStatsMapKeys::CacheId() { + static const std::string kCacheId = "id"; + return kCacheId; +} + +std::string BlockCacheCfStatsMapKeys::UsedBytes(CacheEntryRole role) { + const static std::string kPrefix = "bytes."; + return GetPrefixedCacheEntryRoleName(kPrefix, role); +} + namespace { struct Registry { diff --git a/cache/cache_test.cc b/cache/cache_test.cc index 1a8bae4df5..0526c9ea9e 100644 --- a/cache/cache_test.cc +++ b/cache/cache_test.cc @@ -942,7 +942,8 @@ TEST_P(CacheTest, ApplyToAllCacheEntriesTest) { TEST_P(CacheTest, ApplyToAllEntriesTest) { std::vector callback_state; const auto callback = [&](const Slice& key, void* value, size_t charge, - Cache::DeleterFn deleter) { + Cache::DeleterFn deleter, + Cache::ItemOwnerId /* item_owner_id */) { callback_state.push_back(std::to_string(DecodeKey(key)) + "," + std::to_string(DecodeValue(value)) + "," + std::to_string(charge)); @@ -986,7 +987,8 @@ TEST_P(CacheTest, ApplyToAllEntriesDuringResize) { // For callback int special_count = 0; const auto callback = [&](const Slice&, void*, size_t charge, - Cache::DeleterFn) { + Cache::DeleterFn, + Cache::ItemOwnerId /* item_owner_id */) { if (charge == static_cast(kSpecialCharge)) { ++special_count; } diff --git a/cache/charged_cache.cc b/cache/charged_cache.cc index a9ff969b81..9adc74378e 100644 --- a/cache/charged_cache.cc +++ b/cache/charged_cache.cc @@ -19,8 +19,10 @@ ChargedCache::ChargedCache(std::shared_ptr cache, Status ChargedCache::Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, Handle** handle, - Priority priority) { - Status s = cache_->Insert(key, value, charge, deleter, handle, priority); + Priority priority, + Cache::ItemOwnerId item_owner_id) { + Status s = cache_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); if (s.ok()) { // Insert may cause the cache entry eviction if the cache is full. So we // directly call the reservation manager to update the total memory used @@ -34,8 +36,10 @@ Status ChargedCache::Insert(const Slice& key, void* value, size_t charge, Status ChargedCache::Insert(const Slice& key, void* value, const CacheItemHelper* helper, size_t charge, - Handle** handle, Priority priority) { - Status s = cache_->Insert(key, value, helper, charge, handle, priority); + Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) { + Status s = cache_->Insert(key, value, helper, charge, handle, priority, + item_owner_id); if (s.ok()) { // Insert may cause the cache entry eviction if the cache is full. So we // directly call the reservation manager to update the total memory used diff --git a/cache/charged_cache.h b/cache/charged_cache.h index 1739e40889..6f9509accc 100644 --- a/cache/charged_cache.h +++ b/cache/charged_cache.h @@ -24,10 +24,13 @@ class ChargedCache : public Cache { ~ChargedCache() override = default; Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, - Handle** handle, Priority priority) override; - Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) override; + Status Insert( + const Slice& key, void* value, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override; Cache::Handle* Lookup(const Slice& key, Statistics* stats) override; Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper, @@ -90,7 +93,8 @@ class ChargedCache : public Cache { void ApplyToAllEntries( const std::function& callback, + Cache::DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const Cache::ApplyToAllEntriesOptions& opts) override { cache_->ApplyToAllEntries(callback, opts); } diff --git a/cache/clock_cache.cc b/cache/clock_cache.cc index 58a7f94bb3..c50a61d7bf 100644 --- a/cache/clock_cache.cc +++ b/cache/clock_cache.cc @@ -970,7 +970,8 @@ void ClockCacheShard::EraseUnRefEntries() { table_.EraseUnRefEntries(); } void ClockCacheShard::ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) { // The state is essentially going to be the starting hash, which works // nicely even if we resize between calls because we use upper-most @@ -995,7 +996,8 @@ void ClockCacheShard::ApplyToSomeEntries( table_.ConstApplyToEntriesRange( [callback](const ClockHandle& h) { - callback(h.KeySlice(), h.value, h.total_charge, h.deleter); + callback(h.KeySlice(), h.value, h.total_charge, h.deleter, + Cache::kUnknownItemId); }, index_begin, index_end, false); } @@ -1035,8 +1037,8 @@ void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) { Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value, size_t charge, Cache::DeleterFn deleter, - Cache::Handle** handle, - Cache::Priority priority) { + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId /* item_owner_id */) { if (UNLIKELY(key.size() != kCacheKeySize)) { return Status::NotSupported("ClockCache only supports key size " + std::to_string(kCacheKeySize) + "B"); diff --git a/cache/clock_cache.h b/cache/clock_cache.h index a68514e36f..69d064a48b 100644 --- a/cache/clock_cache.h +++ b/cache/clock_cache.h @@ -508,7 +508,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard { Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, Cache::DeleterFn deleter, Cache::Handle** handle, - Cache::Priority priority) override; + Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override; Cache::Handle* Lookup(const Slice& key, uint32_t hash) override; @@ -531,7 +532,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard { void ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) override; void EraseUnRefEntries() override; @@ -541,8 +543,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard { // SecondaryCache not yet supported Status Insert(const Slice& key, uint32_t hash, void* value, const Cache::CacheItemHelper* helper, size_t charge, - Cache::Handle** handle, Cache::Priority priority) override { - return Insert(key, hash, value, charge, helper->del_cb, handle, priority); + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override { + return Insert(key, hash, value, charge, helper->del_cb, handle, priority, + item_owner_id); } Cache::Handle* Lookup(const Slice& key, uint32_t hash, diff --git a/cache/fast_lru_cache.cc b/cache/fast_lru_cache.cc index f5f93800d5..76ebb39a10 100644 --- a/cache/fast_lru_cache.cc +++ b/cache/fast_lru_cache.cc @@ -210,7 +210,8 @@ void LRUCacheShard::EraseUnRefEntries() { void LRUCacheShard::ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) { // The state is essentially going to be the starting hash, which works // nicely even if we resize between calls because we use upper-most @@ -238,7 +239,7 @@ void LRUCacheShard::ApplyToSomeEntries( [callback, metadata_charge_policy = metadata_charge_policy_](LRUHandle* h) { callback(h->key(), h->value, h->GetCharge(metadata_charge_policy), - h->deleter); + h->deleter, Cache::kUnknownItemId); }, index_begin, index_end); } @@ -323,7 +324,8 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) { Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, size_t charge, Cache::DeleterFn deleter, Cache::Handle** handle, - Cache::Priority /*priority*/) { + Cache::Priority /*priority*/, + Cache::ItemOwnerId /* item_owner_id */) { if (key.size() != kCacheKeySize) { return Status::NotSupported("FastLRUCache only supports key size " + std::to_string(kCacheKeySize) + "B"); diff --git a/cache/fast_lru_cache.h b/cache/fast_lru_cache.h index c4fe8f582f..3344e23187 100644 --- a/cache/fast_lru_cache.h +++ b/cache/fast_lru_cache.h @@ -337,12 +337,15 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard { // nullptr. Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, Cache::DeleterFn deleter, Cache::Handle** handle, - Cache::Priority priority) override; + Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override; Status Insert(const Slice& key, uint32_t hash, void* value, const Cache::CacheItemHelper* helper, size_t charge, - Cache::Handle** handle, Cache::Priority priority) override { - return Insert(key, hash, value, charge, helper->del_cb, handle, priority); + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override { + return Insert(key, hash, value, charge, helper->del_cb, handle, priority, + item_owner_id); } Cache::Handle* Lookup(const Slice& key, uint32_t hash, @@ -372,7 +375,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard { void ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) override; void EraseUnRefEntries() override; diff --git a/cache/lru_cache.cc b/cache/lru_cache.cc index 96431b5729..974eb6f81e 100644 --- a/cache/lru_cache.cc +++ b/cache/lru_cache.cc @@ -161,7 +161,8 @@ void LRUCacheShard::EraseUnRefEntries() { void LRUCacheShard::ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) { // The state is essentially going to be the starting hash, which works // nicely even if we resize between calls because we use upper-most @@ -192,7 +193,7 @@ void LRUCacheShard::ApplyToSomeEntries( ? h->info_.helper->del_cb : h->info_.deleter; callback(h->key(), h->value, h->GetCharge(metadata_charge_policy), - deleter); + deleter, h->item_owner_id); }, index_begin, index_end); } @@ -468,7 +469,7 @@ void LRUCacheShard::Promote(LRUHandle* e) { e->IsHighPri() ? Cache::Priority::HIGH : Cache::Priority::LOW; s = Insert(e->key(), e->hash, /*value=*/nullptr, 0, /*deleter=*/nullptr, /*helper=*/nullptr, /*handle=*/nullptr, - priority); + priority, e->item_owner_id); } else { e->SetInCache(true); e->SetIsStandalone(false); @@ -679,7 +680,8 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, size_t charge, void (*deleter)(const Slice& key, void* value), const Cache::CacheItemHelper* helper, - Cache::Handle** handle, Cache::Priority priority) { + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) { // Allocate the memory here outside of the mutex. // If the cache is full, we'll have to release it. // It shouldn't happen very often though. @@ -706,6 +708,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, memcpy(e->key_data, key.data(), key.size()); e->CalcTotalCharge(charge, metadata_charge_policy_); + e->item_owner_id = item_owner_id; return InsertItem(e, handle, /* free_handle_on_fail */ true); } diff --git a/cache/lru_cache.h b/cache/lru_cache.h index ba5e78946e..38c04aceae 100644 --- a/cache/lru_cache.h +++ b/cache/lru_cache.h @@ -103,6 +103,8 @@ struct LRUHandle { uint16_t flags; + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId; + #ifdef __SANITIZE_THREAD__ // TSAN can report a false data race on flags, where one thread is writing // to one of the mutable bits and another thread is reading this immutable @@ -359,16 +361,18 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard { // Like Cache methods, but with an extra "hash" parameter. virtual Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, Cache::DeleterFn deleter, - Cache::Handle** handle, - Cache::Priority priority) override { - return Insert(key, hash, value, charge, deleter, nullptr, handle, priority); + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override { + return Insert(key, hash, value, charge, deleter, nullptr, handle, priority, + item_owner_id); } virtual Status Insert(const Slice& key, uint32_t hash, void* value, const Cache::CacheItemHelper* helper, size_t charge, - Cache::Handle** handle, - Cache::Priority priority) override { + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) override { assert(helper); - return Insert(key, hash, value, charge, nullptr, helper, handle, priority); + return Insert(key, hash, value, charge, nullptr, helper, handle, priority, + item_owner_id); } // If helper_cb is null, the values of the following arguments don't matter. virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash, @@ -402,7 +406,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard { virtual void ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) override; virtual void EraseUnRefEntries() override; @@ -432,7 +437,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard { bool free_handle_on_fail); Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, DeleterFn deleter, const Cache::CacheItemHelper* helper, - Cache::Handle** handle, Cache::Priority priority); + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id); // Promote an item looked up from the secondary cache to the LRU cache. // The item may be still in the secondary cache. // It is only inserted into the hash table and not the LRU list, and only diff --git a/cache/lru_cache_test.cc b/cache/lru_cache_test.cc index f42404cce2..05e75deadd 100644 --- a/cache/lru_cache_test.cc +++ b/cache/lru_cache_test.cc @@ -54,14 +54,16 @@ class LRUCacheTest : public testing::Test { } void Insert(const std::string& key, - Cache::Priority priority = Cache::Priority::LOW) { + Cache::Priority priority = Cache::Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) { EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/, - nullptr /*deleter*/, nullptr /*handle*/, - priority)); + nullptr /*deleter*/, nullptr /*handle*/, priority, + item_owner_id)); } - void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { - Insert(std::string(1, key), priority); + void Insert(char key, Cache::Priority priority = Cache::Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) { + Insert(std::string(1, key), priority, item_owner_id); } bool Lookup(const std::string& key) { @@ -395,7 +397,7 @@ class FastLRUCacheTest : public testing::Test { Status Insert(const std::string& key) { return cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/, nullptr /*deleter*/, nullptr /*handle*/, - Cache::Priority::LOW); + Cache::Priority::LOW, Cache::kUnknownItemId); } Status Insert(char key, size_t len) { return Insert(std::string(len, key)); } @@ -530,13 +532,16 @@ class ClockCacheTest : public testing::Test { } Status Insert(const std::string& key, - Cache::Priority priority = Cache::Priority::LOW) { + Cache::Priority priority = Cache::Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) { return shard_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/, - nullptr /*deleter*/, nullptr /*handle*/, priority); + nullptr /*deleter*/, nullptr /*handle*/, priority, + item_owner_id); } - Status Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { - return Insert(std::string(kCacheKeySize, key), priority); + Status Insert(char key, Cache::Priority priority = Cache::Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) { + return Insert(std::string(kCacheKeySize, key), priority, item_owner_id); } Status InsertWithLen(char key, size_t len) { @@ -626,9 +631,9 @@ TEST_F(ClockCacheTest, Limits) { // Single entry charge beyond capacity { - Status s = shard_->Insert(key, 0 /*hash*/, nullptr /*value*/, - 5 /*charge*/, nullptr /*deleter*/, - nullptr /*handle*/, Cache::Priority::LOW); + Status s = shard_->Insert( + key, 0 /*hash*/, nullptr /*value*/, 5 /*charge*/, nullptr /*deleter*/, + nullptr /*handle*/, Cache::Priority::LOW, Cache::kUnknownItemId); if (strict_capacity_limit) { EXPECT_TRUE(s.IsMemoryLimit()); } else { @@ -640,7 +645,8 @@ TEST_F(ClockCacheTest, Limits) { { Cache::Handle* h; ASSERT_OK(shard_->Insert(key, 0 /*hash*/, nullptr /*value*/, 3 /*charge*/, - nullptr /*deleter*/, &h, Cache::Priority::LOW)); + nullptr /*deleter*/, &h, Cache::Priority::LOW, + Cache::kUnknownItemId)); // Try to insert more Status s = Insert('a'); if (strict_capacity_limit) { @@ -662,7 +668,8 @@ TEST_F(ClockCacheTest, Limits) { for (size_t i = 0; i < n && s.ok(); ++i) { EncodeFixed64(&key[0], i); s = shard_->Insert(key, 0 /*hash*/, nullptr /*value*/, 0 /*charge*/, - nullptr /*deleter*/, &ha[i], Cache::Priority::LOW); + nullptr /*deleter*/, &ha[i], Cache::Priority::LOW, + Cache::kUnknownItemId); if (i == 0) { EXPECT_OK(s); } @@ -812,7 +819,7 @@ TEST_F(ClockCacheTest, ClockCounterOverflowTest) { std::string my_key(kCacheKeySize, 'x'); uint32_t my_hash = 42; ASSERT_OK(shard_->Insert(my_key, my_hash, &deleted, 1, IncrementIntDeleter, - &h, Cache::Priority::HIGH)); + &h, Cache::Priority::HIGH, Cache::kUnknownItemId)); // Some large number outstanding shard_->TEST_RefN(h, 123456789); @@ -850,13 +857,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { uint32_t my_hash = 42; Cache::Handle* h1; ASSERT_OK(shard_->Insert(key1, my_hash, &deleted, 1, IncrementIntDeleter, &h1, - Cache::Priority::HIGH)); + Cache::Priority::HIGH, Cache::kUnknownItemId)); Cache::Handle* h2; ASSERT_OK(shard_->Insert(key2, my_hash, &deleted, 1, IncrementIntDeleter, &h2, - Cache::Priority::HIGH)); + Cache::Priority::HIGH, Cache::kUnknownItemId)); Cache::Handle* h3; ASSERT_OK(shard_->Insert(key3, my_hash, &deleted, 1, IncrementIntDeleter, &h3, - Cache::Priority::HIGH)); + Cache::Priority::HIGH, Cache::kUnknownItemId)); // Can repeatedly lookup+release despite the hash collision Cache::Handle* tmp_h; @@ -899,7 +906,8 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { // Also Insert with invisible entry there ASSERT_OK(shard_->Insert(key1, my_hash, &deleted, 1, IncrementIntDeleter, - nullptr, Cache::Priority::HIGH)); + nullptr, Cache::Priority::HIGH, + Cache::kUnknownItemId)); tmp_h = shard_->Lookup(key1, my_hash); // Found but distinct handle ASSERT_NE(nullptr, tmp_h); @@ -2050,15 +2058,20 @@ class LRUCacheWithStat : public LRUCache { ~LRUCacheWithStat() {} Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, - Handle** handle, Priority priority) override { + Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) override { insert_count_++; - return LRUCache::Insert(key, value, charge, deleter, handle, priority); - } - Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) override { + return LRUCache::Insert(key, value, charge, deleter, handle, priority, + item_owner_id); + } + Status Insert( + const Slice& key, void* value, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override { insert_count_++; - return LRUCache::Insert(key, value, helper, charge, handle, priority); + return LRUCache::Insert(key, value, helper, charge, handle, priority, + item_owner_id); } Handle* Lookup(const Slice& key, Statistics* stats) override { lookup_count_++; diff --git a/cache/sharded_cache.cc b/cache/sharded_cache.cc index 3e6d6a4f73..67bc142db9 100644 --- a/cache/sharded_cache.cc +++ b/cache/sharded_cache.cc @@ -57,21 +57,25 @@ void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) { Status ShardedCache::Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, Handle** handle, - Priority priority) { + Priority priority, + Cache::ItemOwnerId item_owner_id) { uint32_t hash = HashSlice(key); return GetShard(Shard(hash)) - ->Insert(key, hash, value, charge, deleter, handle, priority); + ->Insert(key, hash, value, charge, deleter, handle, priority, + item_owner_id); } Status ShardedCache::Insert(const Slice& key, void* value, const CacheItemHelper* helper, size_t charge, - Handle** handle, Priority priority) { + Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) { uint32_t hash = HashSlice(key); if (!helper) { return Status::InvalidArgument(); } return GetShard(Shard(hash)) - ->Insert(key, hash, value, helper, charge, handle, priority); + ->Insert(key, hash, value, helper, charge, handle, priority, + item_owner_id); } Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) { @@ -160,7 +164,8 @@ size_t ShardedCache::GetPinnedUsage() const { void ShardedCache::ApplyToAllEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const ApplyToAllEntriesOptions& opts) { uint32_t num_shards = GetNumShards(); // Iterate over part of each shard, rotating between shards, to diff --git a/cache/sharded_cache.h b/cache/sharded_cache.h index 8713d1dce9..f5af9607fa 100644 --- a/cache/sharded_cache.h +++ b/cache/sharded_cache.h @@ -27,10 +27,12 @@ class CacheShard { using DeleterFn = Cache::DeleterFn; virtual Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, DeleterFn deleter, - Cache::Handle** handle, Cache::Priority priority) = 0; + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) = 0; virtual Status Insert(const Slice& key, uint32_t hash, void* value, const Cache::CacheItemHelper* helper, size_t charge, - Cache::Handle** handle, Cache::Priority priority) = 0; + Cache::Handle** handle, Cache::Priority priority, + Cache::ItemOwnerId item_owner_id) = 0; virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0; virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash, const Cache::CacheItemHelper* helper, @@ -56,7 +58,8 @@ class CacheShard { // completion. virtual void ApplyToSomeEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, uint32_t average_entries_per_lock, uint32_t* state) = 0; virtual void EraseUnRefEntries() = 0; virtual std::string GetPrintableOptions() const { return ""; } @@ -82,12 +85,13 @@ class ShardedCache : public Cache { virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override; virtual Status Insert(const Slice& key, void* value, size_t charge, - DeleterFn deleter, Handle** handle, - Priority priority) override; - virtual Status Insert(const Slice& key, void* value, - const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + DeleterFn deleter, Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) override; + virtual Status Insert( + const Slice& key, void* value, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = kUnknownItemId) override; virtual Handle* Lookup(const Slice& key, Statistics* stats) override; virtual Handle* Lookup(const Slice& key, const CacheItemHelper* helper, const CreateCallback& create_cb, Priority priority, @@ -109,7 +113,8 @@ class ShardedCache : public Cache { virtual size_t GetTableAddressCount() const override; virtual void ApplyToAllEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const ApplyToAllEntriesOptions& opts) override; virtual void EraseUnRefEntries() override; virtual std::string GetPrintableOptions() const override; diff --git a/db/column_family.cc b/db/column_family.cc index 62b1ef1a72..4aca2a55b9 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -657,6 +657,11 @@ ColumnFamilyData::ColumnFamilyData( CacheReservationManagerImpl>( bbto->block_cache))); } + + if (bbto->block_cache && table_cache_) { + cache_owner_id_ = bbto->block_cache->GetNextItemOwnerId(); + table_cache_->SetBlockCacheOwnerId(cache_owner_id_); + } } } @@ -669,6 +674,12 @@ ColumnFamilyData::~ColumnFamilyData() { prev->next_ = next; next->prev_ = prev; + const BlockBasedTableOptions* bbto = + ioptions_.table_factory->GetOptions(); + if (bbto && bbto->block_cache) { + bbto->block_cache->DiscardItemOwnerId(&cache_owner_id_); + } + if (!dropped_ && column_family_set_ != nullptr) { // If it's dropped, it's already removed from column family set // If column_family_set_ == nullptr, this is dummy CFD and not in diff --git a/db/column_family.h b/db/column_family.h index de94c544bd..753fe0b651 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -21,6 +21,7 @@ #include "db/write_batch_internal.h" #include "db/write_controller.h" #include "options/cf_options.h" +#include "rocksdb/cache.h" #include "rocksdb/compaction_job_stats.h" #include "rocksdb/db.h" #include "rocksdb/env.h" @@ -562,6 +563,8 @@ class ColumnFamilyData { static constexpr uint64_t kLaggingFlushesThreshold = 10U; void SetNumTimedQueuedForFlush(uint64_t num) { num_queued_for_flush_ = num; } + Cache::ItemOwnerId GetCacheOwnerId() const { return cache_owner_id_; } + private: friend class ColumnFamilySet; ColumnFamilyData(uint32_t id, const std::string& name, @@ -666,6 +669,8 @@ class ColumnFamilyData { // Used in the WBM's flush initiation heuristics. // See DBImpl::InitiateMemoryManagerFlushRequest() for more details uint64_t num_queued_for_flush_ = 0U; + + Cache::ItemOwnerId cache_owner_id_ = Cache::kUnknownItemId; }; // ColumnFamilySet has interesting thread-safety requirements diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index 1486e412e0..eab2e13e9a 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -3476,12 +3476,14 @@ class DBBasicTestMultiGet : public DBTestBase { const char* Name() const override { return "MyBlockCache"; } using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { + Status Insert( + const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value), + Handle** handle = nullptr, Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override { num_inserts_++; - return target_->Insert(key, value, charge, deleter, handle, priority); + return target_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); } using Cache::Lookup; diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index 6c335febce..6b3c39605c 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -35,6 +35,12 @@ namespace ROCKSDB_NAMESPACE { +namespace { +const std::vector CacheCfStatsRoles{ + CacheEntryRole::kDataBlock, CacheEntryRole::kFilterBlock, + CacheEntryRole::kIndexBlock}; +} + class DBBlockCacheTest : public DBTestBase { private: size_t miss_count_ = 0; @@ -168,7 +174,7 @@ class DBBlockCacheTest : public DBTestBase { } #ifndef ROCKSDB_LITE - const std::array GetCacheEntryRoleCountsBg() { + InternalStats::CacheEntryRoleStats GetCacheEntryRoleStatsBg() { // Verify in cache entry role stats ColumnFamilyHandleImpl* cfh = static_cast(dbfull()->DefaultColumnFamily()); @@ -176,9 +182,88 @@ class DBBlockCacheTest : public DBTestBase { InternalStats::CacheEntryRoleStats stats; internal_stats_ptr->TEST_GetCacheEntryRoleStats(&stats, /*foreground=*/false); - return stats.entry_counts; + return stats; + } + + const std::array GetCacheEntryRoleCountsBg() { + return GetCacheEntryRoleStatsBg().entry_counts; } #endif // ROCKSDB_LITE + + void ValidateCacheCfMapProperty( + const std::vector& cf_handles, + const InternalStats::CacheEntryRoleStats& actual_stats) { + // Get the general block cache entry stats using the default cf as + // we are using only the total used bytes which is the total for all + // cf-s in this DB + std::map entry_values; + ASSERT_TRUE(db_->GetMapProperty(dbfull()->DefaultColumnFamily(), + DB::Properties::kBlockCacheEntryStats, + &entry_values)); + for (auto role : CacheCfStatsRoles) { + uint64_t total_role_charges_all_cfs_cf_stats = 0U; + + for (const auto cf_handle : cf_handles) { + ColumnFamilyHandleImpl* cfh = + static_cast(cf_handle); + + std::map cf_values; + ASSERT_TRUE(db_->GetMapProperty(cfh, DB::Properties::kBlockCacheCfStats, + &cf_values)); + + ASSERT_EQ(cfh->GetName(), + cf_values[BlockCacheCfStatsMapKeys::CfName()]); + ASSERT_EQ(actual_stats.cache_id, + cf_values[BlockCacheCfStatsMapKeys::CacheId()]); + + total_role_charges_all_cfs_cf_stats += + std::stoll(cf_values[BlockCacheCfStatsMapKeys::UsedBytes(role)]); + } + + auto total_role_charges_global_stats = + std::stoll(entry_values[BlockCacheCfStatsMapKeys::UsedBytes(role)]); + ASSERT_EQ(total_role_charges_global_stats, + total_role_charges_all_cfs_cf_stats) + << "Role: " << GetCacheEntryRoleName(role); + } + } + +#define CALL_WRAPPER(func) \ + func; \ + ASSERT_FALSE(HasFailure()); + + void ValidateCacheStats( + const std::shared_ptr& cache, + const std::array& expected_counts) { + auto actual_stats = GetCacheEntryRoleStatsBg(); + + auto actual_counts = actual_stats.entry_counts; + EXPECT_EQ(expected_counts, actual_counts); + + std::vector cf_handles(handles_); + if (cf_handles.empty()) { + cf_handles.push_back(dbfull()->DefaultColumnFamily()); + }; + + if (std::string(cache->Name()) == "LRUCache") { + // For LRU block cache, verify that the per-item owner id counts + // are maintained correctly. + // This feature is currently only supported in the LRU cache + for (auto role : CacheCfStatsRoles) { + auto role_idx = static_cast(role); + size_t total_role_charges_all_cfs = 0U; + for (const auto cfh : cf_handles) { + auto cfh_impl = static_cast(cfh); + auto cache_owner_id = cfh_impl->cfd()->GetCacheOwnerId(); + total_role_charges_all_cfs += + actual_stats.charge_per_item_owner[cache_owner_id][role_idx]; + } + ASSERT_EQ(actual_stats.total_charges[role_idx], + total_role_charges_all_cfs); + } + ValidateCacheCfMapProperty(cf_handles, actual_stats); + } + } }; TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) { @@ -406,7 +491,8 @@ class ReadOnlyCacheWrapper : public CacheWrapper { using Cache::Insert; Status Insert(const Slice& /*key*/, void* /*value*/, size_t /*charge*/, void (*)(const Slice& key, void* value) /*deleter*/, - Handle** /*handle*/, Priority /*priority*/) override { + Handle** /*handle*/, Priority /*priority*/, + Cache::ItemOwnerId /* item_owner_id */) override { return Status::NotSupported(); } }; @@ -827,14 +913,16 @@ class MockCache : public LRUCache { Status Insert(const Slice& key, void* value, const Cache::CacheItemHelper* helper_cb, size_t charge, - Handle** handle, Priority priority) override { + Handle** handle, Priority priority, + Cache::ItemOwnerId item_owner_id) override { DeleterFn delete_cb = helper_cb->del_cb; if (priority == Priority::LOW) { low_pri_insert_count++; } else { high_pri_insert_count++; } - return LRUCache::Insert(key, value, charge, delete_cb, handle, priority); + return LRUCache::Insert(key, value, charge, delete_cb, handle, priority, + item_owner_id); } }; @@ -1275,17 +1363,22 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) { } } -static void ClearCache(Cache* cache) { +static void ClearCache(Cache* cache, Cache::ItemOwnerId owner_id_to_clear = + Cache::kUnknownItemId) { auto roles = CopyCacheDeleterRoleMap(); std::deque keys; Cache::ApplyToAllEntriesOptions opts; auto callback = [&](const Slice& key, void* /*value*/, size_t /*charge*/, - Cache::DeleterFn deleter) { + Cache::DeleterFn deleter, + Cache::ItemOwnerId item_owner_id) { if (roles.find(deleter) == roles.end()) { // Keep the stats collector return; } - keys.push_back(key.ToString()); + if ((owner_id_to_clear == Cache::kUnknownItemId) || + (item_owner_id == owner_id_to_clear)) { + keys.push_back(key.ToString()); + } }; cache->ApplyToAllEntries(callback, opts); for (auto& k : keys) { @@ -1353,8 +1446,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { std::array expected{}; // For CacheEntryStatsCollector expected[static_cast(CacheEntryRole::kMisc)] = 1; - EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); - + CALL_WRAPPER(ValidateCacheStats(cache, expected)); std::array prev_expected = expected; // First access only filters @@ -1364,13 +1456,13 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { expected[static_cast(CacheEntryRole::kFilterMetaBlock)] += 2; } // Within some time window, we will get cached entry stats - EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, prev_expected)); // Not enough to force a miss env_->MockSleepForSeconds(45); - EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, prev_expected)); // Enough to force a miss env_->MockSleepForSeconds(601); - EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); // Now access index and data block ASSERT_EQ("value", Get("foo")); @@ -1392,7 +1484,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { env_->MockSleepForSeconds(20); }); SyncPoint::GetInstance()->EnableProcessing(); - EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); prev_expected = expected; SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); @@ -1408,10 +1500,10 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { // Because of the simulated long scan, this is not enough to force // a miss env_->MockSleepForSeconds(601); - EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, prev_expected)); // But this is enough env_->MockSleepForSeconds(10000); - EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); prev_expected = expected; // Also check the GetProperty interface @@ -1472,11 +1564,11 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { // For Fill-it-up expected[static_cast(CacheEntryRole::kMisc)]++; // Still able to hit on saved stats - EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + CALL_WRAPPER(ValidateCacheStats(cache, prev_expected)); + // Enough to force a miss env_->MockSleepForSeconds(1000); - EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); - + CALL_WRAPPER(ValidateCacheStats(cache, expected)); cache->Release(h); // Now we test that the DB mutex is not held during scans, for the ways @@ -1525,8 +1617,211 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { } } +TEST_F(DBBlockCacheTest, CacheStatsPerCfMultipleCfs) { + const size_t capacity = size_t{1} << 25; + auto cache{NewLRUCache(capacity)}; + + Options options = CurrentOptions(); + SetTimeElapseOnlySleepOnReopen(&options); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.max_open_files = 13; + options.table_cache_numshardbits = 0; + // If this wakes up, it could interfere with test + options.stats_dump_period_sec = 0; + + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(50)); + table_options.metadata_cache_options.top_level_index_pinning = + PinningTier::kNone; + table_options.metadata_cache_options.partition_pinning = PinningTier::kNone; + table_options.metadata_cache_options.unpartitioned_pinning = + PinningTier::kNone; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + CreateAndReopenWithCF({"CF1"}, options); + + // Create a new table. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put("bar", "value")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + ASSERT_OK(Put(1, "zfoo", "value")); + ASSERT_OK(Put(1, "zbar", "value")); + ASSERT_OK(Flush(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(0, 1)); + + // Fresh cache + ClearCache(cache.get()); + + std::array expected{}; + // For CacheEntryStatsCollector + expected[static_cast(CacheEntryRole::kMisc)] = 1; + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + // First access only filters + ASSERT_EQ("NOT_FOUND", Get("different from any key added")); + ASSERT_EQ("NOT_FOUND", Get(1, "different from any key added")); + expected[static_cast(CacheEntryRole::kFilterBlock)] += 2; + // Enough to force a miss + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + // Now access index and data block + ASSERT_EQ("value", Get("foo")); + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + expected[static_cast(CacheEntryRole::kDataBlock)]++; + // Enough to force a miss + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + // The same for other CF + ASSERT_EQ("value", Get(1, "zfoo")); + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + expected[static_cast(CacheEntryRole::kDataBlock)]++; + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + auto cf1_owner_id = static_cast(handles_[1]) + ->cfd() + ->GetCacheOwnerId(); + + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_.erase(handles_.begin() + 1); + + --expected[static_cast(CacheEntryRole::kFilterBlock)]; + --expected[static_cast(CacheEntryRole::kIndexBlock)]; + --expected[static_cast(CacheEntryRole::kDataBlock)]; + + // The cache may have items of CF1 in its LRU which will + // be counted => remove them explicitly + ClearCache(cache.get(), cf1_owner_id); + + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + ClearCache(cache.get()); + std::fill(expected.begin(), expected.end(), 0); + // For CacheEntryStatsCollector + expected[static_cast(CacheEntryRole::kMisc)] = 1; + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); + + // Add some more CF-2 + CreateColumnFamilies({"CF2", "CF3", "CF4"}, options); + + for (auto cf_id = 1U; cf_id < 4U; ++cf_id) { + ASSERT_OK(Put(cf_id, std::string("CF") + std::to_string(cf_id) + "-foo", + "value")); + ASSERT_OK(Flush(cf_id)); + ASSERT_EQ(1, NumTableFilesAtLevel(0, 1)); + } + + // Fresh cache + ClearCache(cache.get()); + + ASSERT_EQ("NOT_FOUND", Get(1, "different from any key added")); + expected[static_cast(CacheEntryRole::kFilterBlock)] += 1; + + ASSERT_EQ("value", Get(2, "CF2-foo")); + expected[static_cast(CacheEntryRole::kFilterBlock)]++; + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + expected[static_cast(CacheEntryRole::kDataBlock)]++; + + env_->MockSleepForSeconds(601); + CALL_WRAPPER(ValidateCacheStats(cache, expected)); +} + #endif // ROCKSDB_LITE +TEST_F(DBBlockCacheTest, ItemIdAllocation) { + const size_t capacity = size_t{1} << 25; + auto cache{NewLRUCache(capacity)}; + + size_t max_num_ids = Cache::kMaxOwnerItemId - Cache::kMinOwnerItemId + 1; + auto expected_num_free_ids = max_num_ids; + + // Allocate 10 id-s + auto expected_next_id = Cache::kMinOwnerItemId; + for (auto i = 0U; i < 10U; ++i) { + ASSERT_EQ(cache->GetNextItemOwnerId(), expected_next_id); + ++expected_next_id; + --expected_num_free_ids; + } + --expected_next_id; + + // Release all 10 allocated id-s in reverse order + Cache::ItemOwnerId to_discard_id = expected_next_id; + for (auto i = 0U; i < 10U; ++i) { + auto temp = to_discard_id; + cache->DiscardItemOwnerId(&temp); + ASSERT_EQ(temp, Cache::kUnknownItemId); + + ASSERT_GT(to_discard_id, 0U); + --to_discard_id; + ++expected_num_free_ids; + } + + // Allocate 10 id-s and expect to get the id-s from the free list + // in the reverse order + ASSERT_EQ(expected_next_id, Cache::kMinOwnerItemId + 9U); + for (auto i = 0U; i < 10U; ++i) { + ASSERT_EQ(cache->GetNextItemOwnerId(), expected_next_id); + ASSERT_GT(expected_next_id, 0U); + --expected_next_id; + --expected_num_free_ids; + } + + ASSERT_EQ(expected_num_free_ids, max_num_ids - 10U); + + // Free list should now be empty + // Exhaust all of the id-s before wrap around + expected_next_id = Cache::kMinOwnerItemId + 10U; + while (expected_num_free_ids > 0U) { + ASSERT_EQ(cache->GetNextItemOwnerId(), expected_next_id); + ++expected_next_id; + --expected_num_free_ids; + } + + // Expecting next allocations to fail + for (auto i = 0U; i < 5U; ++i) { + ASSERT_EQ(cache->GetNextItemOwnerId(), Cache::kUnknownItemId); + } + + // Free some arbitrary id-s + Cache::ItemOwnerId owner_id = 5000U; + cache->DiscardItemOwnerId(&owner_id); + owner_id = 1000; + cache->DiscardItemOwnerId(&owner_id); + owner_id = 3000; + cache->DiscardItemOwnerId(&owner_id); + + // Expect allocations to return id-s in the same order as freed + ASSERT_EQ(cache->GetNextItemOwnerId(), 5000); + ASSERT_EQ(cache->GetNextItemOwnerId(), 1000); + ASSERT_EQ(cache->GetNextItemOwnerId(), 3000); + + // All id-s exhausted again + ASSERT_EQ(cache->GetNextItemOwnerId(), Cache::kUnknownItemId); + + // Verify the max size of the free list + for (auto i = 0U; i < 2 * Cache::kMaxFreeItemOwnersIdListSize; ++i) { + owner_id = Cache::kMinOwnerItemId + i; + cache->DiscardItemOwnerId(&owner_id); + } + + for (auto i = 0U; i < Cache::kMaxFreeItemOwnersIdListSize; ++i) { + ASSERT_EQ(cache->GetNextItemOwnerId(), Cache::kMinOwnerItemId + i); + } + + // All id-s exhausted again + ASSERT_EQ(cache->GetNextItemOwnerId(), Cache::kUnknownItemId); +} + class DBBlockCacheKeyTest : public DBTestBase, public testing::WithParamInterface> { diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 1c4020c976..06eacf2ba5 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -1706,8 +1706,9 @@ template Status TargetCacheChargeTrackingCache::Insert( const Slice& key, void* value, size_t charge, void (*deleter)(const Slice& key, void* value), Handle** handle, - Priority priority) { - Status s = target_->Insert(key, value, charge, deleter, handle, priority); + Priority priority, Cache::ItemOwnerId item_owner_id) { + Status s = target_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); if (deleter == kNoopDeleter) { if (last_peak_tracked_) { cache_charge_peak_ = 0; diff --git a/db/db_test_util.h b/db/db_test_util.h index 0a35d9ffcd..4f640a0d02 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -883,11 +883,13 @@ class CacheWrapper : public Cache { const char* Name() const override { return target_->Name(); } using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { - return target_->Insert(key, value, charge, deleter, handle, priority); + Status Insert( + const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value), Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override { + return target_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); } using Cache::Lookup; @@ -942,7 +944,8 @@ class CacheWrapper : public Cache { void ApplyToAllEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const ApplyToAllEntriesOptions& opts) override { target_->ApplyToAllEntries(callback, opts); } @@ -971,10 +974,11 @@ class TargetCacheChargeTrackingCache : public CacheWrapper { explicit TargetCacheChargeTrackingCache(std::shared_ptr target); using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), - Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + Status Insert( + const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value), Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override; using Cache::Release; bool Release(Handle* handle, bool erase_if_last_ref = false) override; diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 5a8a243249..4cf1f76f90 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -247,6 +247,7 @@ static const std::string cf_file_histogram = "cf-file-histogram"; static const std::string dbstats = "dbstats"; static const std::string levelstats = "levelstats"; static const std::string block_cache_entry_stats = "block-cache-entry-stats"; +static const std::string block_cache_cf_stats = "block-cache-cf-stats"; static const std::string num_immutable_mem_table = "num-immutable-mem-table"; static const std::string num_immutable_mem_table_flushed = "num-immutable-mem-table-flushed"; @@ -326,6 +327,8 @@ const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats; const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats; const std::string DB::Properties::kBlockCacheEntryStats = rocksdb_prefix + block_cache_entry_stats; +const std::string DB::Properties::kBlockCacheCfStats = + rocksdb_prefix + block_cache_cf_stats; const std::string DB::Properties::kNumImmutableMemTable = rocksdb_prefix + num_immutable_mem_table; const std::string DB::Properties::kNumImmutableMemTableFlushed = @@ -446,6 +449,9 @@ const UnorderedMap {DB::Properties::kBlockCacheEntryStats, {true, &InternalStats::HandleBlockCacheEntryStats, nullptr, &InternalStats::HandleBlockCacheEntryStatsMap, nullptr}}, + {DB::Properties::kBlockCacheCfStats, + {true, &InternalStats::HandleBlockCacheCfStats, nullptr, + &InternalStats::HandleBlockCacheCfStatsMap, nullptr}}, {DB::Properties::kSSTables, {false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}}, {DB::Properties::kAggregatedTableProperties, @@ -644,10 +650,11 @@ void InternalStats::CollectCacheEntryStats(bool foreground) { min_interval_factor); } -std::function +std::function InternalStats::CacheEntryRoleStats::GetEntryCallback() { - return [&](const Slice& /*key*/, void* /*value*/, size_t charge, - Cache::DeleterFn deleter) { + return [&](const Slice& /*key*/, void* /* value */, size_t charge, + Cache::DeleterFn deleter, Cache::ItemOwnerId item_owner_id) { auto e = role_map_.find(deleter); size_t role_idx; if (e == role_map_.end()) { @@ -657,6 +664,7 @@ InternalStats::CacheEntryRoleStats::GetEntryCallback() { } entry_counts[role_idx]++; total_charges[role_idx] += charge; + charge_per_item_owner[item_owner_id][role_idx] += charge; }; } @@ -674,6 +682,8 @@ void InternalStats::CacheEntryRoleStats::BeginCollection( cache_usage = cache->GetUsage(); table_size = cache->GetTableAddressCount(); occupancy = cache->GetOccupancyCount(); + // Currently only LRUCache supports this feature + charge_per_item_owner_supported = (cache->Name() == std::string("LRUCache")); } void InternalStats::CacheEntryRoleStats::EndCollection( @@ -717,6 +727,41 @@ std::string InternalStats::CacheEntryRoleStats::ToString( return str.str(); } +std::string InternalStats::CacheEntryRoleStats::CacheOwnerStatsToString( + const std::string& cf_name, Cache::ItemOwnerId cache_owner_id) { + if (charge_per_item_owner_supported == false) { + return ""; + } + + std::ostringstream str; + const auto& cf_charges_per_role_pos = + charge_per_item_owner.find(cache_owner_id); + + std::vector roles{CacheEntryRole::kDataBlock, + CacheEntryRole::kFilterBlock, + CacheEntryRole::kIndexBlock}; + std::array roles_total_charge{}; + + str << "Block cache [" << cf_name << "] "; + + if (cf_charges_per_role_pos != charge_per_item_owner.end()) { + const std::array& cf_charges_per_role = + cf_charges_per_role_pos->second; + for (auto role : roles) { + auto role_idx = static_cast(role); + roles_total_charge[role_idx] = cf_charges_per_role[role_idx]; + } + } + + for (auto role : roles) { + auto role_idx = static_cast(role); + str << " " << kCacheEntryRoleToCamelString[role_idx] << "(" + << BytesToHumanString(roles_total_charge[role_idx]) << ")"; + } + str << '\n'; + return str.str(); +} + void InternalStats::CacheEntryRoleStats::ToMap( std::map* values, SystemClock* clock) const { values->clear(); @@ -739,6 +784,29 @@ void InternalStats::CacheEntryRoleStats::ToMap( } } +void InternalStats::CacheEntryRoleStats::CacheOwnerStatsToMap( + const std::string& cf_name, Cache::ItemOwnerId cache_owner_id, + std::map* values) const { + if (charge_per_item_owner_supported == false) { + return; + } + + values->clear(); + auto& v = *values; + v[BlockCacheCfStatsMapKeys::CfName()] = cf_name; + v[BlockCacheCfStatsMapKeys::CacheId()] = cache_id; + const auto& cache_owner_charges = charge_per_item_owner.find(cache_owner_id); + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + auto role = static_cast(i); + if (cache_owner_charges != charge_per_item_owner.end()) { + v[BlockCacheCfStatsMapKeys::UsedBytes(role)] = + std::to_string(charge_per_item_owner.at(cache_owner_id)[i]); + } else { + v[BlockCacheCfStatsMapKeys::UsedBytes(role)] = "0"; + } + } +} + bool InternalStats::HandleBlockCacheEntryStats(std::string* value, Slice /*suffix*/) { if (!cache_entry_stats_collector_) { @@ -763,6 +831,31 @@ bool InternalStats::HandleBlockCacheEntryStatsMap( return true; } +bool InternalStats::HandleBlockCacheCfStats(std::string* value, + Slice /*suffix*/) { + if (!cache_entry_stats_collector_) { + return false; + } + CollectCacheEntryStats(/*foreground*/ true); + CacheEntryRoleStats stats; + cache_entry_stats_collector_->GetStats(&stats); + *value = + stats.CacheOwnerStatsToString(cfd_->GetName(), cfd_->GetCacheOwnerId()); + return true; +} + +bool InternalStats::HandleBlockCacheCfStatsMap( + std::map* values, Slice /*suffix*/) { + if (!cache_entry_stats_collector_) { + return false; + } + CollectCacheEntryStats(/*foreground*/ true); + CacheEntryRoleStats stats; + cache_entry_stats_collector_->GetStats(&stats); + stats.CacheOwnerStatsToMap(cfd_->GetName(), cfd_->GetCacheOwnerId(), values); + return true; +} + bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix) { uint64_t temperature; @@ -1892,6 +1985,8 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) { // Skip if stats are extremely old (> 1 day, incl not yet populated) if (now_micros - stats.last_end_time_micros_ < kDayInMicros) { value->append(stats.ToString(clock_)); + value->append(stats.CacheOwnerStatsToString(cfd_->GetName(), + cfd_->GetCacheOwnerId())); } } } diff --git a/db/internal_stats.h b/db/internal_stats.h index 7091877bb1..5f85543fc4 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -10,9 +10,11 @@ #pragma once +#include #include #include #include +#include #include #include "cache/cache_entry_roles.h" @@ -463,6 +465,14 @@ class InternalStats { uint32_t copies_of_last_collection = 0; uint64_t last_start_time_micros_ = 0; uint64_t last_end_time_micros_ = 0; + // Mapping of the total charge of all items per owner + bool charge_per_item_owner_supported = false; + // Instances of this class are created per cf, but cache stats collection + // is expensive and cf-agnostic anyway. Therefore, we store the values + // for all cf-s in every instance. + std::unordered_map> + charge_per_item_owner; void Clear() { // Wipe everything except collection_count @@ -472,7 +482,8 @@ class InternalStats { } void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros); - std::function + std::function GetEntryCallback(); void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros); void SkippedCollection(); @@ -481,6 +492,12 @@ class InternalStats { void ToMap(std::map* values, SystemClock* clock) const; + std::string CacheOwnerStatsToString(const std::string& cf_name, + Cache::ItemOwnerId cache_owner_id); + void CacheOwnerStatsToMap(const std::string& cf_name, + Cache::ItemOwnerId cache_owner_id, + std::map* values) const; + private: UnorderedMap role_map_; uint64_t GetLastDurationMicros() const; @@ -796,6 +813,11 @@ class InternalStats { bool HandleBlockCacheEntryStats(std::string* value, Slice suffix); bool HandleBlockCacheEntryStatsMap(std::map* values, Slice suffix); + + bool HandleBlockCacheCfStats(std::string* value, Slice suffix); + bool HandleBlockCacheCfStatsMap(std::map* values, + Slice suffix); + bool HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix); bool HandleNumBlobFiles(uint64_t* value, DBImpl* db, Version* version); bool HandleBlobStats(std::string* value, Slice suffix); diff --git a/db/table_cache.cc b/db/table_cache.cc index 756417cc3b..800cc47c7d 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -162,15 +162,17 @@ Status TableCache::GetTableReader( } else { expected_unique_id = kNullUniqueId64x2; // null ID == no verification } + TableReaderOptions table_reader_options( + ioptions_, prefix_extractor, file_options, internal_comparator, + skip_filters, immortal_tables_, false /* force_direct_prefetch */, + level, block_cache_tracer_, max_file_size_for_l0_meta_pin, + db_session_id_, file_meta.fd.GetNumber(), expected_unique_id, + file_meta.fd.largest_seqno); + table_reader_options.cache_owner_id = cache_owner_id_; + s = ioptions_.table_factory->NewTableReader( - ro, - TableReaderOptions(ioptions_, prefix_extractor, file_options, - internal_comparator, skip_filters, immortal_tables_, - false /* force_direct_prefetch */, level, - block_cache_tracer_, max_file_size_for_l0_meta_pin, - db_session_id_, file_meta.fd.GetNumber(), - expected_unique_id, file_meta.fd.largest_seqno), - std::move(file_reader), file_meta.fd.GetFileSize(), table_reader, + ro, table_reader_options, std::move(file_reader), + file_meta.fd.GetFileSize(), table_reader, prefetch_index_and_filter_in_cache); TEST_SYNC_POINT("TableCache::GetTableReader:0"); } diff --git a/db/table_cache.h b/db/table_cache.h index 2e50f2c779..edf6febc1c 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -229,6 +229,10 @@ class TableCache { } } + void SetBlockCacheOwnerId(Cache::ItemOwnerId cache_owner_id) { + cache_owner_id_ = cache_owner_id; + } + private: // Build a table reader Status GetTableReader( @@ -270,6 +274,7 @@ class TableCache { Striped loader_mutex_; std::shared_ptr io_tracer_; std::string db_session_id_; + Cache::ItemOwnerId cache_owner_id_ = Cache::kUnknownItemId; }; } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/cache.h b/include/rocksdb/cache.h index 658c5485f1..bb8decbfa3 100644 --- a/include/rocksdb/cache.h +++ b/include/rocksdb/cache.h @@ -22,9 +22,12 @@ #pragma once +#include #include #include -#include +#include +#include +#include #include #include "rocksdb/compression_type.h" @@ -304,6 +307,12 @@ class Cache { // are prioritized over them. enum class Priority { HIGH, LOW, BOTTOM }; + using ItemOwnerId = uint16_t; + static constexpr ItemOwnerId kUnknownItemId = 0U; + static constexpr ItemOwnerId kMinOwnerItemId = 1U; + static constexpr ItemOwnerId kMaxOwnerItemId = + std::numeric_limits::max(); + // A set of callbacks to allow objects in the primary block cache to be // be persisted in a secondary cache. The purpose of the secondary cache // is to support other ways of caching the object, such as persistent or @@ -416,9 +425,11 @@ class Cache { // value will be passed to "deleter" which must delete the value. // (The Cache is responsible for copying and reclaiming space for // the key.) + static constexpr Priority kDefaultPriority = Priority::LOW; virtual Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, Handle** handle = nullptr, - Priority priority = Priority::LOW) = 0; + Priority priority = kDefaultPriority, + Cache::ItemOwnerId item_owner_id = kUnknownItemId) = 0; // If the cache has no mapping for "key", returns nullptr. // @@ -534,16 +545,18 @@ class Cache { // also. virtual void ApplyToAllEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const ApplyToAllEntriesOptions& opts) = 0; // DEPRECATED version of above. (Default implementation uses above.) virtual void ApplyToAllCacheEntries(void (*callback)(void* value, size_t charge), bool /*thread_safe*/) { - ApplyToAllEntries([callback](const Slice&, void* value, size_t charge, - DeleterFn) { callback(value, charge); }, - {}); + ApplyToAllEntries( + [callback](const Slice&, void* value, size_t charge, DeleterFn, + uint64_t) { callback(value, charge); }, + {}); } // Remove all entries. @@ -596,11 +609,13 @@ class Cache { virtual Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) { + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = kUnknownItemId) { if (!helper) { return Status::InvalidArgument(); } - return Insert(key, value, charge, helper->del_cb, handle, priority); + return Insert(key, value, charge, helper->del_cb, handle, priority, + item_owner_id); } // Lookup the key in the primary and secondary caches (if one is configured). @@ -656,8 +671,28 @@ class Cache { // to each of the handles. virtual void WaitAll(std::vector& /*handles*/) {} + ItemOwnerId GetNextItemOwnerId(); + // On return will set the owner id to kUnknownItemId + void DiscardItemOwnerId(ItemOwnerId*); + + static constexpr size_t kMaxFreeItemOwnersIdListSize = 10000U; + private: std::shared_ptr memory_allocator_; + + class ItemOwnerIdAllocator { + public: + ItemOwnerId Allocate(); + void Free(ItemOwnerId* id); + + private: + ItemOwnerId next_item_owner_id_ = kMinOwnerItemId; + bool has_wrapped_around_ = false; + std::mutex free_ids_mutex_; + std::list free_ids_; + }; + + ItemOwnerIdAllocator owner_id_allocator_; }; // Classifications of block cache entries. @@ -720,4 +755,13 @@ struct BlockCacheEntryStatsMapKeys { static std::string UsedPercent(CacheEntryRole); }; +// For use with `GetMapProperty()` for property +// `DB::Properties::kBlockCacheCfStats`. On success, the map will +// be populated with all keys that can be obtained from these functions. +struct BlockCacheCfStatsMapKeys { + static const std::string& CfName(); + static const std::string& CacheId(); + static std::string UsedBytes(CacheEntryRole); +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 9359a45ba0..a9916d49da 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -914,6 +914,10 @@ class DB { // available in the map form. static const std::string kBlockCacheEntryStats; + // "rocksdb.block-cache-cf-stats" - returns a multi-line string + // with statistics on block cache usage for a specific column-family. + static const std::string kBlockCacheCfStats; + // "rocksdb.num-immutable-mem-table" - returns number of immutable // memtables that have not yet been flushed. static const std::string kNumImmutableMemTable; diff --git a/plugin/speedb/paired_filter/speedb_db_bloom_filter_test.cc b/plugin/speedb/paired_filter/speedb_db_bloom_filter_test.cc index 2cba4ba7a9..3f6acd1fe0 100644 --- a/plugin/speedb/paired_filter/speedb_db_bloom_filter_test.cc +++ b/plugin/speedb/paired_filter/speedb_db_bloom_filter_test.cc @@ -895,11 +895,13 @@ class FilterConstructResPeakTrackingCache : public CacheWrapper { cache_res_increments_sum_(0) {} using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { - Status s = target_->Insert(key, value, charge, deleter, handle, priority); + Status Insert( + const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value), Handle** handle = nullptr, + Priority priority = Priority::LOW, + Cache::ItemOwnerId item_owner_id = Cache::kUnknownItemId) override { + Status s = target_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); if (deleter == kNoopDeleterForFilterConstruction) { if (last_peak_tracked_) { cache_res_peak_ = 0; diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index da81cb254c..60f3d92ee5 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -1516,7 +1516,7 @@ Status BlockBasedTableBuilder::InsertBlockInCache(const Slice& block_contents, s = block_cache->Insert( key.AsSlice(), block_holder.get(), BlocklikeTraits::GetCacheItemHelper(block_type), charge, - nullptr, Cache::Priority::LOW); + nullptr, Cache::Priority::LOW, rep_->props.column_family_id); if (s.ok()) { // Release ownership of block_holder. diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index a0bdc9a2d0..f10a891ad7 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -629,7 +629,7 @@ Status BlockBasedTableFactory::NewTableReader( table_reader_options.block_cache_tracer, table_reader_options.max_file_size_for_l0_meta_pin, table_reader_options.cur_db_session_id, table_reader_options.cur_file_num, - table_reader_options.unique_id); + table_reader_options.unique_id, table_reader_options.cache_owner_id); } TableBuilder* BlockBasedTableFactory::NewTableBuilder( diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index 75c016576b..2d538e7d9a 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -417,10 +417,11 @@ Status BlockBasedTable::InsertEntryToCache( Status s = Status::OK(); if (cache_tier == CacheTier::kNonVolatileBlockTier) { s = block_cache->Insert(key, block_holder.get(), cache_helper, charge, - cache_handle, priority); + cache_handle, priority, rep_->cache_owner_id); } else { s = block_cache->Insert(key, block_holder.get(), charge, - cache_helper->del_cb, cache_handle, priority); + cache_helper->del_cb, cache_handle, priority, + rep_->cache_owner_id); } return s; } @@ -589,7 +590,8 @@ Status BlockBasedTable::Open( TailPrefetchStats* tail_prefetch_stats, BlockCacheTracer* const block_cache_tracer, size_t max_file_size_for_l0_meta_pin, const std::string& cur_db_session_id, - uint64_t cur_file_num, UniqueId64x2 expected_unique_id) { + uint64_t cur_file_num, UniqueId64x2 expected_unique_id, + Cache::ItemOwnerId cache_owner_id) { table_reader->reset(); Status s; @@ -649,9 +651,9 @@ Status BlockBasedTable::Open( } BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; - Rep* rep = new BlockBasedTable::Rep(ioptions, env_options, table_options, - internal_comparator, skip_filters, - file_size, level, immortal_table); + Rep* rep = new BlockBasedTable::Rep( + ioptions, env_options, table_options, internal_comparator, skip_filters, + file_size, level, immortal_table, cache_owner_id); rep->file = std::move(file); rep->footer = footer; diff --git a/table/block_based/block_based_table_reader.h b/table/block_based/block_based_table_reader.h index 8667ab43dc..142b50ae49 100644 --- a/table/block_based/block_based_table_reader.h +++ b/table/block_based/block_based_table_reader.h @@ -109,7 +109,8 @@ class BlockBasedTable : public TableReader { BlockCacheTracer* const block_cache_tracer = nullptr, size_t max_file_size_for_l0_meta_pin = 0, const std::string& cur_db_session_id = "", uint64_t cur_file_num = 0, - UniqueId64x2 expected_unique_id = {}); + UniqueId64x2 expected_unique_id = {}, + Cache::ItemOwnerId cache_owner_id = Cache::kUnknownItemId); bool PrefixRangeMayMatch(const Slice& internal_key, const ReadOptions& read_options, @@ -550,7 +551,8 @@ struct BlockBasedTable::Rep { Rep(const ImmutableOptions& _ioptions, const EnvOptions& _env_options, const BlockBasedTableOptions& _table_opt, const InternalKeyComparator& _internal_comparator, bool skip_filters, - uint64_t _file_size, int _level, const bool _immortal_table) + uint64_t _file_size, int _level, const bool _immortal_table, + Cache::ItemOwnerId _cache_owner_id = Cache::kUnknownItemId) : ioptions(_ioptions), env_options(_env_options), table_options(_table_opt), @@ -563,7 +565,8 @@ struct BlockBasedTable::Rep { global_seqno(kDisableGlobalSequenceNumber), file_size(_file_size), level(_level), - immortal_table(_immortal_table) {} + immortal_table(_immortal_table), + cache_owner_id(_cache_owner_id) {} ~Rep() { status.PermitUncheckedError(); } const ImmutableOptions& ioptions; const EnvOptions& env_options; @@ -629,6 +632,7 @@ struct BlockBasedTable::Rep { bool index_value_is_full = true; const bool immortal_table; + Cache::ItemOwnerId cache_owner_id = Cache::kUnknownItemId; std::unique_ptr table_reader_cache_res_handle = nullptr; diff --git a/table/block_based/block_based_table_reader_impl.h b/table/block_based/block_based_table_reader_impl.h index dc321567a3..eed9aef209 100644 --- a/table/block_based/block_based_table_reader_impl.h +++ b/table/block_based/block_based_table_reader_impl.h @@ -95,7 +95,8 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( CacheKey key = CacheKey::CreateUniqueForCacheLifetime(block_cache); s = block_cache->Insert(key.AsSlice(), nullptr, block.GetValue()->ApproximateMemoryUsage(), - nullptr, &cache_handle); + nullptr, &cache_handle, Cache::kDefaultPriority, + rep_->table_properties->column_family_id); if (s.ok()) { assert(cache_handle != nullptr); @@ -153,7 +154,8 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(const ReadOptions& ro, CacheKey key = CacheKey::CreateUniqueForCacheLifetime(block_cache); s = block_cache->Insert(key.AsSlice(), nullptr, block.GetValue()->ApproximateMemoryUsage(), - nullptr, &cache_handle); + nullptr, &cache_handle, Cache::kDefaultPriority, + rep_->table_properties->column_family_id); if (s.ok()) { assert(cache_handle != nullptr); diff --git a/table/table_builder.h b/table/table_builder.h index 1790f33b1b..c6786d5b82 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -86,6 +86,8 @@ struct TableReaderOptions { // Known unique_id or {}, kNullUniqueId64x2 means unknown UniqueId64x2 unique_id; + + Cache::ItemOwnerId cache_owner_id = Cache::kUnknownItemId; }; struct TableBuilderOptions { diff --git a/utilities/cache_dump_load_impl.cc b/utilities/cache_dump_load_impl.cc index 33b0d37ec4..e639b50063 100644 --- a/utilities/cache_dump_load_impl.cc +++ b/utilities/cache_dump_load_impl.cc @@ -107,10 +107,10 @@ bool CacheDumperImpl::ShouldFilterOut(const Slice& key) { // Cache::ApplyToAllEntries. In this callback function, we will get the block // type, decide if the block needs to be dumped based on the filter, and write // the block through the provided writer. -std::function +std::function CacheDumperImpl::DumpOneBlockCallBack() { return [&](const Slice& key, void* value, size_t /*charge*/, - Cache::DeleterFn deleter) { + Cache::DeleterFn deleter, Cache::ItemOwnerId /* item_owner_id */) { // Step 1: get the type of the block from role_map_ auto e = role_map_.find(deleter); CacheEntryRole role; diff --git a/utilities/cache_dump_load_impl.h b/utilities/cache_dump_load_impl.h index ad8cd045a9..b2b95f3c3b 100644 --- a/utilities/cache_dump_load_impl.h +++ b/utilities/cache_dump_load_impl.h @@ -113,7 +113,7 @@ class CacheDumperImpl : public CacheDumper { void* value, size_t len); IOStatus WriteFooter(); bool ShouldFilterOut(const Slice& key); - std::function + std::function DumpOneBlockCallBack(); CacheDumpOptions options_; diff --git a/utilities/simulator_cache/sim_cache.cc b/utilities/simulator_cache/sim_cache.cc index a883b52e78..f763ec6832 100644 --- a/utilities/simulator_cache/sim_cache.cc +++ b/utilities/simulator_cache/sim_cache.cc @@ -167,7 +167,7 @@ class SimCacheImpl : public SimCache { using Cache::Insert; Status Insert(const Slice& key, void* value, size_t charge, void (*deleter)(const Slice& key, void* value), Handle** handle, - Priority priority) override { + Priority priority, Cache::ItemOwnerId item_owner_id) override { // The handle and value passed in are for real cache, so we pass nullptr // to key_only_cache_ for both instead. Also, the deleter function pointer // will be called by user to perform some external operation which should @@ -178,7 +178,7 @@ class SimCacheImpl : public SimCache { // TODO: Check for error here? auto s = key_only_cache_->Insert( key, nullptr, charge, [](const Slice& /*k*/, void* /*v*/) {}, nullptr, - priority); + priority, item_owner_id); s.PermitUncheckedError(); } else { key_only_cache_->Release(h); @@ -188,7 +188,8 @@ class SimCacheImpl : public SimCache { if (!cache_) { return Status::OK(); } - return cache_->Insert(key, value, charge, deleter, handle, priority); + return cache_->Insert(key, value, charge, deleter, handle, priority, + item_owner_id); } using Cache::Lookup; @@ -261,7 +262,8 @@ class SimCacheImpl : public SimCache { void ApplyToAllEntries( const std::function& callback, + DeleterFn deleter, + Cache::ItemOwnerId item_owner_id)>& callback, const ApplyToAllEntriesOptions& opts) override { cache_->ApplyToAllEntries(callback, opts); }