Snapshot on each block

src/Makefile.am

@@ -204,6 +204,7 @@ DEFI_CORE_H = \
   dfi/oracles.h \
   dfi/poolpairs.h \
   dfi/proposals.h \
+  dfi/snapshotmanager.h \
   dfi/tokens.h \
   dfi/threadpool.h \
   dfi/coinselect.h \
@@ -461,6 +462,7 @@ libdefi_server_a_SOURCES = \
   dfi/rpc_tokens.cpp \
   dfi/rpc_vault.cpp \
   dfi/skipped_txs.cpp \
+  dfi/snapshotmanager.cpp \
   dfi/tokens.cpp \
   dfi/threadpool.cpp \
   dfi/undos.cpp \

src/dbwrapper.h

@@ -362,8 +362,12 @@ class CDBWrapper
         return new CDBIterator(*this, pdb->NewIterator(readOptions));
     }
 
-    [[nodiscard]] std::shared_ptr<CStorageSnapshot> GetStorageSnapshot() const {
-        return std::make_shared<CStorageSnapshot>(pdb);
+    [[nodiscard]] const leveldb::Snapshot* GetLevelDBSnapshot() const {
+        return pdb->GetSnapshot();
+    }
+
+    void ReleaseSnapshot(const leveldb::Snapshot* snapshot) const {
+        pdb->ReleaseSnapshot(snapshot);
     }
 
     /**

src/dfi/masternodes.cpp

@@ -1399,11 +1399,3 @@ void CalcMissingRewardTempFix(CCustomCSView &mnview, const uint32_t targetHeight
         }
     }
 }
-
-std::unique_ptr<CCustomCSView> GetViewSnapshot() {
-    // Get database snapshot and flushable storage changed map
-    auto [changed, snapshotDB] = pcustomcsview->GetStorage().GetSnapshotPair();
-
-    // Create new view using snapshot and change map
-    return std::make_unique<CCustomCSView>(snapshotDB, changed);
-}

src/dfi/masternodes.h

@@ -628,8 +628,6 @@ class CCustomCSView : public CMasternodesView,
 
 std::map<CKeyID, CKey> AmISignerNow(int height, const CAnchorData::CTeam &team);
 
-std::unique_ptr<CCustomCSView> GetViewSnapshot();
-
 /** Global DB and view that holds enhanced chainstate data (should be protected by cs_main) */
 extern std::unique_ptr<CStorageLevelDB> pcustomcsDB;
 extern std::unique_ptr<CCustomCSView> pcustomcsview;

src/dfi/snapshotmanager.cpp

@@ -0,0 +1,123 @@
+#include <dfi/snapshotmanager.h>
+
+#include <dfi/masternodes.h>
+
+std::unique_ptr<CCustomCSView> GetViewSnapshot() {
+    MapKV changed;
+    std::unique_ptr<CStorageLevelDB> snapshotDB;
+
+    // Get database snapshot and flushable storage changed map
+    const auto res = psnapshotManager->CheckoutSnapshot(changed, snapshotDB);
+
+    // If snapshot not present get global snapshot
+    if (!res) {
+        // Get snapshot and changed map from global
+        psnapshotManager->GetGlobalSnapshot(changed, snapshotDB);
+
+        // Create new view using snapshot and change map
+        return std::make_unique<CCustomCSView>(snapshotDB, changed);
+    }
+
+    // Create new view using snapshot and change map
+    return std::make_unique<CCustomCSView>(snapshotDB, changed);
+}
+
+CCheckedOutSnapshot::~CCheckedOutSnapshot() {
+    // Check snapshot back in
+    psnapshotManager->ReturnSnapshot(key);
+}
+
+CSnapshotManager::CSnapshotManager(CStorageLevelDB *other) {
+    db = other;
+}
+
+void CSnapshotManager::SetBlockSnapshot(CCustomCSView &view, const CBlockIndex *block) {
+    std::unique_lock<std::mutex> lock(mtx);
+
+    // Get database snapshot and flushable storage changed map
+    auto [changed, snapshot] = view.GetStorage().GetSnapshotData();
+
+    // Release current entry if not in use
+    if (currentSnapshot && currentSnapshot->snapshot && !checkedOutMap.count(currentSnapshot->key)) {
+        db->GetDB()->ReleaseSnapshot(currentSnapshot->snapshot);
+    }
+
+    // Set current snapshot
+    currentSnapshot =
+        std::make_unique<CBlockSnapshot>(snapshot, changed, CBlockSnapshotKey{block->nHeight, block->GetBlockHash()});
+}
+
+void CSnapshotManager::GetGlobalSnapshot(MapKV &changed, std::unique_ptr<CStorageLevelDB> &snapshotDB) {
+    LOCK(cs_main);
+
+    // Get database snapshot and flushable storage changed map
+    auto [changedMap, snapshot] = pcustomcsview->GetStorage().GetSnapshotData();
+
+    // Create checked out snapshot
+    auto checkedSnapshot = std::make_unique<CCheckedOutSnapshot>(snapshot, CBlockSnapshotKey{});
+
+    // Set args
+    changed = changedMap;
+    snapshotDB = std::make_unique<CStorageLevelDB>(db->GetDB(), checkedSnapshot);
+}
+
+bool CSnapshotManager::CheckoutSnapshot(MapKV &changed, std::unique_ptr<CStorageLevelDB> &snapshotDB) {
+    std::unique_lock<std::mutex> lock(mtx);
+
+    if (!currentSnapshot) {
+        return false;
+    }
+
+    // Track checked out snapshot
+    if (checkedOutMap.count(currentSnapshot->key)) {
+        ++checkedOutMap.at(currentSnapshot->key).count;
+    } else {
+        checkedOutMap[currentSnapshot->key] = {currentSnapshot->snapshot, 1};
+    }
+
+    // Create checked out snapshot
+    auto snapshot = std::make_unique<CCheckedOutSnapshot>(currentSnapshot->snapshot, currentSnapshot->key);
+
+    // Set args
+    changed = currentSnapshot->changed;
+    snapshotDB = std::make_unique<CStorageLevelDB>(db->GetDB(), snapshot);
+
+    return true;
+}
+
+void CSnapshotManager::EraseCurrentSnapshot() {
+    std::unique_lock<std::mutex> lock(mtx);
+
+    if (!currentSnapshot) {
+        return;
+    }
+
+    if (!checkedOutMap.count(currentSnapshot->key)) {
+        db->GetDB()->ReleaseSnapshot(currentSnapshot->snapshot);
+    }
+
+    currentSnapshot.reset();
+}
+
+void CSnapshotManager::ReturnSnapshot(const CBlockSnapshotKey &key) {
+    std::unique_lock<std::mutex> lock(mtx);
+
+    if (!checkedOutMap.count(key)) {
+        return;
+    }
+
+    --checkedOutMap.at(key).count;
+
+    bool isCurrentKey{};
+    if (currentSnapshot) {
+        isCurrentKey = currentSnapshot->key.hash == key.hash;
+    }
+
+    // Release if not in use and not the current block
+    if (!checkedOutMap.count(key) && !isCurrentKey) {
+        db->GetDB()->ReleaseSnapshot(checkedOutMap.at(key).snapshot);
+        checkedOutMap.erase(key);
+    }
+}
+
+std::unique_ptr<CSnapshotManager> psnapshotManager;

src/dfi/snapshotmanager.h

@@ -0,0 +1,98 @@
+#ifndef DEFI_DFI_SNAPSHOTMANAGER_H
+#define DEFI_DFI_SNAPSHOTMANAGER_H
+
+#include <uint256.h>
+
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <vector>
+
+class CBlockIndex;
+class CCustomCSView;
+class CSnapshotManager;
+class CStorageLevelDB;
+
+namespace leveldb {
+    class Snapshot;
+}
+
+using TBytes = std::vector<unsigned char>;
+using MapKV = std::map<TBytes, std::optional<TBytes>>;
+
+std::unique_ptr<CCustomCSView> GetViewSnapshot();
+
+struct CBlockSnapshotKey {
+    int64_t height{};
+    uint256 hash{};
+
+    struct Comparator {
+        bool operator()(const CBlockSnapshotKey &a, const CBlockSnapshotKey &b) const {
+            if (a.height < b.height) {
+                return true;
+            }
+            if (a.height > b.height) {
+                return false;
+            }
+            return a.hash < b.hash;
+        }
+    };
+};
+
+struct CBlockSnapshotValue {
+    const leveldb::Snapshot *snapshot;
+    int64_t count;
+};
+
+struct CBlockSnapshot {
+    const leveldb::Snapshot *snapshot{};
+    MapKV changed;
+    CBlockSnapshotKey key;
+
+    CBlockSnapshot(const leveldb::Snapshot *snapshot, MapKV changed, const CBlockSnapshotKey &key)
+        : snapshot(snapshot),
+          changed(std::move(changed)),
+          key(key) {}
+};
+
+class CCheckedOutSnapshot {
+public:
+    explicit CCheckedOutSnapshot(const leveldb::Snapshot *other, const CBlockSnapshotKey &otherKey)
+        : snapshot(other),
+          key(otherKey) {}
+    CCheckedOutSnapshot(const CCheckedOutSnapshot &) = delete;
+    CCheckedOutSnapshot &operator=(const CCheckedOutSnapshot &) = delete;
+
+    ~CCheckedOutSnapshot();
+
+    [[nodiscard]] const leveldb::Snapshot *GetLevelDBSnapshot() const { return snapshot; }
+
+private:
+    const leveldb::Snapshot *snapshot;
+    CBlockSnapshotKey key;
+};
+
+class CSnapshotManager {
+    std::unique_ptr<CBlockSnapshot> currentSnapshot;
+    std::mutex mtx;
+    CStorageLevelDB *db;
+    std::map<CBlockSnapshotKey, CBlockSnapshotValue, CBlockSnapshotKey::Comparator> checkedOutMap;
+
+public:
+    CSnapshotManager() = delete;
+    explicit CSnapshotManager(CStorageLevelDB *other);
+
+    CSnapshotManager(const CSnapshotManager &other) = delete;
+    CSnapshotManager &operator=(const CSnapshotManager &other) = delete;
+
+    void SetBlockSnapshot(CCustomCSView &view, const CBlockIndex *block);
+    bool CheckoutSnapshot(MapKV &changed, std::unique_ptr<CStorageLevelDB> &snapshotDB);
+    void GetGlobalSnapshot(MapKV &changed, std::unique_ptr<CStorageLevelDB> &snapshotDB);
+    void ReturnSnapshot(const CBlockSnapshotKey &key);
+    void EraseCurrentSnapshot();
+};
+
+extern std::unique_ptr<CSnapshotManager> psnapshotManager;
+
+#endif  // DEFI_DFI_SNAPSHOTMANAGER_H

src/flushablestorage.h

@@ -5,6 +5,8 @@
 #ifndef DEFI_FLUSHABLESTORAGE_H
 #define DEFI_FLUSHABLESTORAGE_H
 
+#include <dfi/snapshotmanager.h>
+
 #include <shutdown.h>
 
 #include <dbwrapper.h>
@@ -142,8 +144,8 @@ class CStorageLevelDB : public CStorageKV {
         : db{std::make_shared<CDBWrapper>(dbName, cacheSize, fMemory, fWipe)}, batch(*db) {}
 
     // Snapshot constructor
-    CStorageLevelDB(std::shared_ptr<CDBWrapper> &db, std::shared_ptr<CStorageSnapshot> &otherSnapshot)
-            : db(db), batch(*db), snapshot(otherSnapshot) {
+    CStorageLevelDB(std::shared_ptr<CDBWrapper> &db, std::unique_ptr<CCheckedOutSnapshot> &otherSnapshot)
+            : db(db), batch(*db), snapshot(std::move(otherSnapshot)) {
         options.snapshot = snapshot->GetLevelDBSnapshot();
     }
 
@@ -201,9 +203,10 @@ class CStorageLevelDB : public CStorageKV {
         return db->IsEmpty();
     }
 
-    [[nodiscard]] std::shared_ptr<CStorageSnapshot> GetStorageSnapshot() const {
-        return db->GetStorageSnapshot();
+    [[nodiscard]] const leveldb::Snapshot* GetLevelDBSnapshot() const {
+        return db->GetLevelDBSnapshot();
     }
+
     [[nodiscard]] std::shared_ptr<CDBWrapper>& GetDB() {
         return db;
     }
@@ -215,7 +218,7 @@ class CStorageLevelDB : public CStorageKV {
 
     // If this snapshot is set it will be used when
     // reading from the DB.
-    std::shared_ptr<CStorageSnapshot> snapshot;
+    std::unique_ptr<CCheckedOutSnapshot> snapshot;
 };
 
 // Flushable storage
@@ -367,43 +370,21 @@ class CFlushableStorageKV : public CStorageKV {
         return changed;
     }
 
-    std::pair<MapKV, std::unique_ptr<CStorageLevelDB>> GetSnapshotPair() {
+    [[nodiscard]] CStorageLevelDB* GetStorageLevelDB() const {
         const auto storageLevelDB = dynamic_cast<CStorageLevelDB*>(&db);
         assert(storageLevelDB);
-        if (blockTipChanged.load()) {
-            // Lock cs_main when updating from tipSnapshot to avoid
-            // race with Dis/ConnectTip.
-            LOCK(cs_main);
-            // Double check bool for safety now we are under lock.
-            if (blockTipChanged.load()) {
-                tipSnapshot = storageLevelDB->GetStorageSnapshot();
-                changedCopy = changed;
-                blockTipChanged.store(false);
-            }
-        }
-
-        return {changedCopy, std::make_unique<CStorageLevelDB>(storageLevelDB->GetDB(), tipSnapshot)};
+        return storageLevelDB;
     }
 
-    void BlockTipChanged() {
-        blockTipChanged.store(true);
+    std::pair<MapKV, const leveldb::Snapshot*> GetSnapshotData() {
+        return {changed, GetStorageLevelDB()->GetLevelDBSnapshot()};
     }
 
 private:
     std::unique_ptr<CStorageLevelDB> snapshotDB;
     CStorageKV& db;
     MapKV changed;
 
-    // Used to create a CStorageLevelDB object.
-    std::shared_ptr<CStorageSnapshot> tipSnapshot;
-
-    // Copy of the changed map at the point of snapshot creation.
-    // Can be used lock free.
-    MapKV changedCopy;
-
-    // Used to determine whether the block tip has changed.
-    std::atomic<bool> blockTipChanged{true};
-
     // Whether this view is using a snapshot
     bool snapshot{};
 };

src/init.cpp

@@ -2267,6 +2267,10 @@ bool AppInitMain(InitInterfaces& interfaces)
         block_notify_genesis_wait_connection.disconnect();
     }
 
+    // Set snapshot now chain has loaded
+    psnapshotManager = std::make_unique<CSnapshotManager>(pcustomcsview->GetStorage().GetStorageLevelDB());
+
+
     if (ShutdownRequested()) {
         return false;
     }

src/rpc/resultcache.cpp

@@ -1,6 +1,7 @@
 #include <rpc/resultcache.h>
 #include <rpc/util.h>
 #include <logging.h>
+#include <validation.h>
 
 void RPCResultCache::Init(RPCCacheMode mode) {
     std::unique_lock l{aMutex};