Merge pull request #153 from nanocurrency/unchecked_memory

This change puts unchecked blocks in to a memory container instead of disk when the initial bootstrap threshold is reached. Blocks are removed from the container in fifo order once the maximum of 256,000 blocks are reached.

nano/core_test/block_store.cpp

@@ -482,9 +482,11 @@ TEST (block_store, empty_bootstrap)
 	auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
 	ASSERT_TRUE (!store->init_error ());
 	auto transaction (store->tx_begin_read ());
-	auto begin (store->unchecked.begin (transaction));
-	auto end (store->unchecked.end ());
-	ASSERT_EQ (end, begin);
+	bool has_item = false;
+	store->unchecked.for_each (transaction, [&has_item] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+		has_item = true;
+	});
+	ASSERT_FALSE (has_item);
 }
 
 TEST (block_store, one_bootstrap)
@@ -495,17 +497,16 @@ TEST (block_store, one_bootstrap)
 	auto block1 (std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5));
 	auto transaction (store->tx_begin_write ());
 	store->unchecked.put (transaction, block1->hash (), block1);
-	auto begin (store->unchecked.begin (transaction));
-	auto end (store->unchecked.end ());
-	ASSERT_NE (end, begin);
-	auto hash1 (begin->first.key ());
-	ASSERT_EQ (block1->hash (), hash1);
-	auto blocks (store->unchecked.get (transaction, hash1));
+	std::vector<nano::hash_or_account> dependencies;
+	store->unchecked.for_each (transaction, [&dependencies] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+		dependencies.push_back (key.key ());
+	});
+	std::vector<nano::hash_or_account> expected{ block1->hash () };
+	ASSERT_EQ (dependencies, expected);
+	auto blocks (store->unchecked.get (transaction, block1->hash ()));
 	ASSERT_EQ (1, blocks.size ());
 	auto block2 (blocks[0].block);
 	ASSERT_EQ (*block1, *block2);
-	++begin;
-	ASSERT_EQ (end, begin);
 }
 
 TEST (block_store, unchecked_begin_search)
@@ -930,32 +931,30 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
 	ASSERT_NE (send1->hash (), send2->hash ());
 	store.unchecked.put (transaction, send1->hash (), send1);
 	store.unchecked.put (transaction, send1->hash (), send2);
-	{
-		auto iterator1 (store.unchecked.begin (transaction));
-		++iterator1;
-		ASSERT_EQ (store.unchecked.end (), iterator1);
-	}
+	std::vector<nano::hash_or_account> dependencies1;
+	store.unchecked.for_each (transaction, [&dependencies1] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+		dependencies1.push_back (key.key ());
+	});
+	ASSERT_TRUE (dependencies1.empty ());
 	ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 0));
 	mdb_dbi_close (store.env, store.unchecked_handle);
 	ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
 	store.unchecked.put (transaction, send1->hash (), send1);
 	store.unchecked.put (transaction, send1->hash (), send2);
-	{
-		auto iterator1 (store.unchecked.begin (transaction));
-		++iterator1;
-		ASSERT_EQ (store.unchecked.end (), iterator1);
-	}
+	std::vector<nano::hash_or_account> dependencies2;
+	store.unchecked.for_each (transaction, [&dependencies2] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+		dependencies2.push_back (key.key ());
+	});
+	ASSERT_TRUE (dependencies2.empty ());
 	ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1));
 	ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
 	store.unchecked.put (transaction, send1->hash (), send1);
 	store.unchecked.put (transaction, send1->hash (), send2);
-	{
-		auto iterator1 (store.unchecked.begin (transaction));
-		++iterator1;
-		ASSERT_NE (store.unchecked.end (), iterator1);
-		++iterator1;
-		ASSERT_EQ (store.unchecked.end (), iterator1);
-	}
+	std::vector<nano::hash_or_account> dependencies3;
+	store.unchecked.for_each (transaction, [&dependencies3] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+		dependencies3.push_back (key.key ());
+	});
+	ASSERT_EQ (1, dependencies3.size ());
 }
 
 TEST (block_store, state_block)

nano/core_test/ledger.cpp

@@ -3690,11 +3690,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
 	ASSERT_EQ (confirmation_height_info.frontier, send->hash ());
 	ASSERT_TRUE (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ())).size () == 1);
 	ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ()))[0], nano::block_hash (2));
-
-	auto unchecked_infos = rocksdb_store.unchecked.get (rocksdb_transaction, nano::dev::genesis->hash ());
-	ASSERT_EQ (unchecked_infos.size (), 1);
-	ASSERT_EQ (unchecked_infos.front ().account, nano::dev::genesis->account ());
-	ASSERT_EQ (*unchecked_infos.front ().block, *send);
 }
 
 TEST (ledger, unconfirmed_frontiers)

nano/nano_node/entry.cpp

@@ -437,14 +437,13 @@ int main (int argc, char * const * argv)
 			}
 
 			// Check all unchecked keys for matching frontier hashes. Indicates an issue with process_batch algorithm
-			for (auto i (node->store.unchecked.begin (transaction)), n (node->store.unchecked.end ()); i != n; ++i)
-			{
-				auto it = frontier_hashes.find (i->first.key ());
+			node->store.unchecked.for_each (transaction, [&frontier_hashes] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
+				auto it = frontier_hashes.find (key.key ());
 				if (it != frontier_hashes.cend ())
 				{
 					std::cout << it->to_string () << "\n";
 				}
-			}
+			});
 		}
 		else if (vm.count ("debug_account_count"))
 		{

nano/node/json_handler.cpp

@@ -4026,9 +4026,8 @@ void nano::json_handler::unchecked ()
 	{
 		boost::property_tree::ptree unchecked;
 		auto transaction (node.store.tx_begin_read ());
-		for (auto i (node.store.unchecked.begin (transaction)), n (node.store.unchecked.end ()); i != n && unchecked.size () < count; ++i)
-		{
-			nano::unchecked_info const & info (i->second);
+		node.store.unchecked.for_each (
+		transaction, [&unchecked, &json_block_l] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
 			if (json_block_l)
 			{
 				boost::property_tree::ptree block_node_l;
@@ -4040,8 +4039,7 @@ void nano::json_handler::unchecked ()
 				std::string contents;
 				info.block->serialize_json (contents);
 				unchecked.put (info.block->hash ().to_string (), contents);
-			}
-		}
+			} }, [iterations = 0, count = count] () mutable { return iterations++ < count; });
 		response_l.add_child ("blocks", unchecked);
 	}
 	response_errors ();
@@ -4063,13 +4061,11 @@ void nano::json_handler::unchecked_get ()
 	auto hash (hash_impl ());
 	if (!ec)
 	{
-		auto transaction (node.store.tx_begin_read ());
-		for (auto i (node.store.unchecked.begin (transaction)), n (node.store.unchecked.end ()); i != n; ++i)
-		{
-			nano::unchecked_key const & key (i->first);
+		bool done = false;
+		node.store.unchecked.for_each (
+		node.store.tx_begin_read (), [&] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
 			if (key.hash == hash)
 			{
-				nano::unchecked_info const & info (i->second);
 				response_l.put ("modified_timestamp", std::to_string (info.modified));
 
 				if (json_block_l)
@@ -4084,9 +4080,8 @@ void nano::json_handler::unchecked_get ()
 					info.block->serialize_json (contents);
 					response_l.put ("contents", contents);
 				}
-				break;
-			}
-		}
+				done = true;
+			} }, [&] () { return !done; });
 		if (response_l.empty ())
 		{
 			ec = nano::error_blocks::not_found;
@@ -4112,11 +4107,10 @@ void nano::json_handler::unchecked_keys ()
 	{
 		boost::property_tree::ptree unchecked;
 		auto transaction (node.store.tx_begin_read ());
-		for (auto i (node.store.unchecked.begin (transaction, nano::unchecked_key (key, 0))), n (node.store.unchecked.end ()); i != n && unchecked.size () < count; ++i)
-		{
+		node.store.unchecked.for_each (
+		transaction, key, [&unchecked, json_block_l] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
 			boost::property_tree::ptree entry;
-			nano::unchecked_info const & info (i->second);
-			entry.put ("key", i->first.key ().to_string ());
+			entry.put ("key", key.key ().to_string ());
 			entry.put ("hash", info.block->hash ().to_string ());
 			entry.put ("modified_timestamp", std::to_string (info.modified));
 			if (json_block_l)
@@ -4131,8 +4125,7 @@ void nano::json_handler::unchecked_keys ()
 				info.block->serialize_json (contents);
 				entry.put ("contents", contents);
 			}
-			unchecked.push_back (std::make_pair ("", entry));
-		}
+			unchecked.push_back (std::make_pair ("", entry)); }, [&unchecked, &count] () { return unchecked.size () < count; });
 		response_l.add_child ("unchecked", unchecked);
 	}
 	response_errors ();

nano/node/lmdb/lmdb.cpp

@@ -811,17 +811,6 @@ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesyst
 	}
 }
 
-std::vector<nano::unchecked_info> nano::unchecked_mdb_store::get (nano::transaction const & transaction_a, nano::block_hash const & hash_a)
-{
-	std::vector<nano::unchecked_info> result;
-	for (auto i (begin (transaction_a, nano::unchecked_key (hash_a, 0))), n (end ()); i != n && i->first.key () == hash_a; ++i)
-	{
-		nano::unchecked_info const & unchecked_info (i->second);
-		result.push_back (unchecked_info);
-	}
-	return result;
-}
-
 nano::unchecked_mdb_store::unchecked_mdb_store (nano::mdb_store & mdb_store_a) :
 	unchecked_store_partial<MDB_val, mdb_store> (mdb_store_a){};
 

nano/node/lmdb/lmdb.hpp

@@ -45,7 +45,6 @@ class unchecked_mdb_store : public unchecked_store_partial<MDB_val, mdb_store>
 {
 public:
 	explicit unchecked_mdb_store (nano::mdb_store &);
-	std::vector<nano::unchecked_info> get (nano::transaction const &, nano::block_hash const &);
 };
 
 /**

nano/node/node.cpp

@@ -147,6 +147,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
 	startup_time (std::chrono::steady_clock::now ()),
 	node_seq (seq)
 {
+	store.unchecked.use_memory = [this] () { return ledger.bootstrap_weight_reached (); };
 	if (!init_error ())
 	{
 		telemetry->start ();
@@ -947,16 +948,13 @@ void nano::node::unchecked_cleanup ()
 		auto const now (nano::seconds_since_epoch ());
 		auto const transaction (store.tx_begin_read ());
 		// Max 1M records to clean, max 2 minutes reading to prevent slow i/o systems issues
-		for (auto i (store.unchecked.begin (transaction)), n (store.unchecked.end ()); i != n && cleaning_list.size () < 1024 * 1024 && nano::seconds_since_epoch () - now < 120; ++i)
-		{
-			nano::unchecked_key const & key (i->first);
-			nano::unchecked_info const & info (i->second);
+		store.unchecked.for_each (
+		transaction, [this, &digests, &cleaning_list, &now] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
 			if ((now - info.modified) > static_cast<uint64_t> (config.unchecked_cutoff_time.count ()))
 			{
 				digests.push_back (network.publish_filter.hash (info.block));
 				cleaning_list.push_back (key);
-			}
-		}
+			} }, [iterations = 0, count = 1024 * 1024] () mutable { return iterations++ < count; });
 	}
 	if (!cleaning_list.empty ())
 	{

nano/node/rocksdb/rocksdb.cpp

@@ -624,44 +624,6 @@ nano::version_rocksdb_store::version_rocksdb_store (nano::rocksdb_store & rocksd
 	nano::version_store_partial<rocksdb::Slice, nano::rocksdb_store> (rocksdb_store_a),
 	rocksdb_store{ rocksdb_store_a } {};
 
-std::vector<nano::unchecked_info> nano::unchecked_rocksdb_store::get (nano::transaction const & transaction_a, nano::block_hash const & hash_a)
-{
-	auto cf = rocksdb_store.table_to_column_family (tables::unchecked);
-
-	std::unique_ptr<rocksdb::Iterator> iter;
-	nano::qualified_root upper (hash_a, nano::block_hash (std::numeric_limits<nano::uint256_t>::max ()));
-	nano::rocksdb_val upper_bound (sizeof (upper), (void *)&upper);
-	if (is_read (transaction_a))
-	{
-		auto read_options = snapshot_options (transaction_a);
-		read_options.prefix_same_as_start = true;
-		read_options.auto_prefix_mode = true;
-		read_options.iterate_upper_bound = upper_bound;
-		read_options.fill_cache = false;
-		iter.reset (rocksdb_store.db->NewIterator (read_options, cf));
-	}
-	else
-	{
-		rocksdb::ReadOptions read_options;
-		read_options.prefix_same_as_start = true;
-		read_options.auto_prefix_mode = true;
-		read_options.iterate_upper_bound = upper_bound;
-		read_options.fill_cache = false;
-		iter.reset (rocksdb_store.tx (transaction_a)->GetIterator (read_options, cf));
-	}
-
-	// Uses prefix extraction
-	std::vector<nano::unchecked_info> result;
-
-	auto prefix = nano::rocksdb_val (hash_a);
-	for (iter->Seek (prefix); iter->Valid () && iter->key ().starts_with (prefix); iter->Next ())
-	{
-		auto unchecked_info = static_cast<nano::unchecked_info> (nano::rocksdb_val (iter->value ()));
-		result.push_back (unchecked_info);
-	}
-	return result;
-}
-
 void nano::rocksdb_store::construct_column_family_mutexes ()
 {
 	for (auto table : all_tables ())

nano/node/rocksdb/rocksdb.hpp

@@ -35,7 +35,6 @@ class unchecked_rocksdb_store : public unchecked_store_partial<rocksdb::Slice, n
 {
 public:
 	explicit unchecked_rocksdb_store (nano::rocksdb_store &);
-	std::vector<nano::unchecked_info> get (nano::transaction const &, nano::block_hash const &);
 
 private:
 	nano::rocksdb_store & rocksdb_store;

nano/secure/common.cpp

@@ -894,6 +894,11 @@ bool nano::unchecked_key::operator== (nano::unchecked_key const & other_a) const
 	return previous == other_a.previous && hash == other_a.hash;
 }
 
+bool nano::unchecked_key::operator< (nano::unchecked_key const & other_a) const
+{
+	return previous != other_a.previous ? previous < other_a.previous : hash < other_a.hash;
+}
+
 nano::block_hash const & nano::unchecked_key::key () const
 {
 	return previous;

nano/secure/common.hpp

@@ -175,6 +175,7 @@ class unchecked_key final
 	unchecked_key (nano::uint512_union const &);
 	bool deserialize (nano::stream &);
 	bool operator== (nano::unchecked_key const &) const;
+	bool operator< (nano::unchecked_key const &) const;
 	nano::block_hash const & key () const;
 	nano::block_hash previous{ 0 };
 	nano::block_hash hash{ 0 };

nano/secure/ledger.cpp

@@ -1438,15 +1438,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
 			}
 		});
 
-		store.unchecked.for_each_par (
-		[&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) {
-			for (; i != n; ++i)
-			{
-				auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::unchecked }));
-				rocksdb_store->unchecked.put (rocksdb_transaction, i->first, i->second);
-			}
-		});
-
 		store.pending.for_each_par (
 		[&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) {
 			for (; i != n; ++i)
@@ -1517,7 +1508,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
 		}
 
 		// Compare counts
-		error |= store.unchecked.count (lmdb_transaction) != rocksdb_store->unchecked.count (rocksdb_transaction);
 		error |= store.peer.count (lmdb_transaction) != rocksdb_store->peer.count (rocksdb_transaction);
 		error |= store.pruned.count (lmdb_transaction) != rocksdb_store->pruned.count (rocksdb_transaction);
 		error |= store.final_vote.count (lmdb_transaction) != rocksdb_store->final_vote.count (rocksdb_transaction);

nano/secure/store.hpp

@@ -743,17 +743,22 @@ class confirmation_height_store
 class unchecked_store
 {
 public:
+	virtual ~unchecked_store () = default;
 	virtual void clear (nano::write_transaction const &) = 0;
 	virtual void put (nano::write_transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) = 0;
 	virtual void put (nano::write_transaction const &, nano::block_hash const &, std::shared_ptr<nano::block> const &) = 0;
 	virtual std::vector<nano::unchecked_info> get (nano::transaction const &, nano::block_hash const &) = 0;
 	virtual bool exists (nano::transaction const & transaction_a, nano::unchecked_key const & unchecked_key_a) = 0;
 	virtual void del (nano::write_transaction const &, nano::unchecked_key const &) = 0;
-	virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> begin (nano::transaction const &) const = 0;
-	virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> begin (nano::transaction const &, nano::unchecked_key const &) const = 0;
-	virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> end () const = 0;
+	virtual void for_each (
+	nano::transaction const & tx, std::function<void (nano::unchecked_key const &, nano::unchecked_info const &)> action, std::function<bool ()> predicate = [] () { return true; })
+	= 0;
+	virtual void for_each (
+	nano::transaction const & tx, nano::hash_or_account const & dependency, std::function<void (nano::unchecked_key const &, nano::unchecked_info const &)> action, std::function<bool ()> predicate = [] () { return true; })
+	= 0;
 	virtual size_t count (nano::transaction const &) = 0;
-	virtual void for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::unchecked_key, nano::unchecked_info>, nano::store_iterator<nano::unchecked_key, nano::unchecked_info>)> const & action_a) const = 0;
+
+	std::function<bool ()> use_memory = [] () { return true; };
 };
 
 /**