diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index ef25d75fac..48322f0f28 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -279,7 +279,9 @@ TEST (block_store, genesis) ASSERT_LE (info.modified, nano::seconds_since_epoch ()); ASSERT_EQ (info.block_count, 1); // Genesis block should be confirmed by default - ASSERT_EQ (info.confirmation_height, 1); + uint64_t confirmation_height; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (confirmation_height, 1); auto test_pub_text (nano::test_genesis_key.pub.to_string ()); auto test_pub_account (nano::test_genesis_key.pub.to_account ()); auto test_prv_text (nano::test_genesis_key.prv.data.to_string ()); @@ -484,8 +486,9 @@ TEST (block_store, frontier_retrieval) auto store = nano::make_store (init, logger, nano::unique_path ()); ASSERT_TRUE (!init); nano::account account1 (0); - nano::account_info info1 (0, 0, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::account_info info1 (0, 0, 0, 0, 0, 0, nano::epoch::epoch_0); auto transaction (store->tx_begin_write ()); + store->confirmation_height_put (transaction, account1, 0); store->account_put (transaction, account1, info1); nano::account_info info2; store->account_get (transaction, account1, info2); @@ -501,7 +504,8 @@ TEST (block_store, one_account) nano::account account (0); nano::block_hash hash (0); auto transaction (store->tx_begin_write ()); - store->account_put (transaction, account, { hash, account, hash, 42, 100, 200, 20, nano::epoch::epoch_0 }); + store->confirmation_height_put (transaction, account, 20); + store->account_put (transaction, account, { hash, account, hash, 42, 100, 200, nano::epoch::epoch_0 }); auto begin (store->latest_begin (transaction)); auto end (store->latest_end ()); ASSERT_NE (end, begin); @@ -511,7 +515,9 @@ TEST (block_store, one_account) ASSERT_EQ (42, info.balance.number ()); ASSERT_EQ (100, info.modified); ASSERT_EQ (200, info.block_count); - ASSERT_EQ (20, info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store->confirmation_height_get (transaction, account, confirmation_height)); + ASSERT_EQ (20, confirmation_height); ++begin; ASSERT_EQ (end, begin); } @@ -551,8 +557,10 @@ TEST (block_store, two_account) nano::account account2 (3); nano::block_hash hash2 (4); auto transaction (store->tx_begin_write ()); - store->account_put (transaction, account1, { hash1, account1, hash1, 42, 100, 300, 20, nano::epoch::epoch_0 }); - store->account_put (transaction, account2, { hash2, account2, hash2, 84, 200, 400, 30, nano::epoch::epoch_0 }); + store->confirmation_height_put (transaction, account1, 20); + store->account_put (transaction, account1, { hash1, account1, hash1, 42, 100, 300, nano::epoch::epoch_0 }); + store->confirmation_height_put (transaction, account2, 30); + store->account_put (transaction, account2, { hash2, account2, hash2, 84, 200, 400, nano::epoch::epoch_0 }); auto begin (store->latest_begin (transaction)); auto end (store->latest_end ()); ASSERT_NE (end, begin); @@ -562,7 +570,9 @@ TEST (block_store, two_account) ASSERT_EQ (42, info1.balance.number ()); ASSERT_EQ (100, info1.modified); ASSERT_EQ (300, info1.block_count); - ASSERT_EQ (20, info1.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store->confirmation_height_get (transaction, account1, confirmation_height)); + ASSERT_EQ (20, confirmation_height); ++begin; ASSERT_NE (end, begin); ASSERT_EQ (account2, nano::account (begin->first)); @@ -571,7 +581,8 @@ TEST (block_store, two_account) ASSERT_EQ (84, info2.balance.number ()); ASSERT_EQ (200, info2.modified); ASSERT_EQ (400, info2.block_count); - ASSERT_EQ (30, info2.confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, account2, confirmation_height)); + ASSERT_EQ (30, confirmation_height); ++begin; ASSERT_EQ (end, begin); } @@ -587,8 +598,10 @@ TEST (block_store, latest_find) nano::account account2 (3); nano::block_hash hash2 (4); auto transaction (store->tx_begin_write ()); - store->account_put (transaction, account1, { hash1, account1, hash1, 100, 0, 300, 0, nano::epoch::epoch_0 }); - store->account_put (transaction, account2, { hash2, account2, hash2, 200, 0, 400, 0, nano::epoch::epoch_0 }); + store->confirmation_height_put (transaction, account1, 0); + store->account_put (transaction, account1, { hash1, account1, hash1, 100, 0, 300, nano::epoch::epoch_0 }); + store->confirmation_height_put (transaction, account2, 0); + store->account_put (transaction, account2, { hash2, account2, hash2, 200, 0, 400, nano::epoch::epoch_0 }); auto first (store->latest_begin (transaction)); auto second (store->latest_begin (transaction)); ++second; @@ -661,6 +674,7 @@ TEST (block_store, latest_exists) nano::block_hash two (2); nano::account_info info; auto transaction (store->tx_begin_write ()); + store->confirmation_height_put (transaction, two, 0); store->account_put (transaction, two, info); nano::block_hash one (1); ASSERT_FALSE (store->account_exists (transaction, one)); @@ -679,6 +693,7 @@ TEST (block_store, large_iteration) nano::account account; nano::random_pool::generate_block (account.bytes.data (), account.bytes.size ()); accounts1.insert (account); + store->confirmation_height_put (transaction, account, 0); store->account_put (transaction, account, nano::account_info ()); } std::unordered_set accounts2; @@ -756,6 +771,7 @@ TEST (block_store, account_count) auto transaction (store->tx_begin_write ()); ASSERT_EQ (0, store->account_count (transaction)); nano::account account (200); + store->confirmation_height_put (transaction, account, 0); store->account_put (transaction, account, nano::account_info ()); } auto transaction (store->tx_begin_read ()); @@ -1605,13 +1621,15 @@ TEST (mdb_block_store, upgrade_v13_v14) store.initialize (transaction, genesis); nano::account_info account_info; ASSERT_FALSE (store.account_get (transaction, nano::genesis_account, account_info)); - ASSERT_EQ (account_info.confirmation_height, 1); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (confirmation_height, 1); store.version_put (transaction, 13); modify_account_info_to_v13 (store, transaction, nano::genesis_account); - // This should fail as sizes are no longer correct for account_info - nano::account_info account_info1; - ASSERT_TRUE (store.account_get (transaction, nano::genesis_account, account_info1)); + // This should fail as sizes are no longer correct for account_info_v14 + nano::mdb_val value; + ASSERT_TRUE (mdb_get (store.env.tx (transaction), store.accounts_v1, nano::mdb_val (nano::genesis_account), value)); } // Now do the upgrade and confirm that confirmation height is 0 and version is updated as expected @@ -1622,9 +1640,9 @@ TEST (mdb_block_store, upgrade_v13_v14) auto transaction (store.tx_begin_write ()); // This should now work and have a confirmation height of 1 - nano::account_info account_info; - ASSERT_FALSE (store.account_get (transaction, nano::genesis_account, account_info)); - ASSERT_EQ (account_info.confirmation_height, 1); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (confirmation_height, 1); ASSERT_LT (13, store.version_get (transaction)); // Test deleting node ID @@ -1643,41 +1661,34 @@ TEST (block_store, confirmation_height) nano::mdb_store store (error, logger, path); nano::account account1 (0); - nano::account_info info1 (0, 0, 0, 0, 0, 0, 500, nano::epoch::epoch_0); nano::account account2 (1); - nano::account_info info2 (0, 0, 0, 0, 0, 0, std::numeric_limits::max (), nano::epoch::epoch_0); nano::account account3 (2); - nano::account_info info3 (0, 0, 0, 0, 0, 0, 10, nano::epoch::epoch_0); - nano::account_info stored_account_info; { auto transaction (store.tx_begin_write ()); - store.account_put (transaction, account1, info1); - store.account_put (transaction, account2, info2); - store.account_put (transaction, account3, info3); - - ASSERT_FALSE (store.account_get (transaction, account1, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, 500); - - ASSERT_FALSE (store.account_get (transaction, account2, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, std::numeric_limits::max ()); - - ASSERT_FALSE (store.account_get (transaction, account3, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, 10); + store.confirmation_height_put (transaction, account1, 500); + store.confirmation_height_put (transaction, account2, std::numeric_limits::max ()); + store.confirmation_height_put (transaction, account3, 10); + + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height)); + ASSERT_EQ (confirmation_height, 500); + ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height)); + ASSERT_EQ (confirmation_height, std::numeric_limits::max ()); + ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height)); + ASSERT_EQ (confirmation_height, 10); // Check cleaning of confirmation heights store.confirmation_height_clear (transaction); } auto transaction (store.tx_begin_read ()); - ASSERT_EQ (store.account_count (transaction), 3); - - ASSERT_FALSE (store.account_get (transaction, account1, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, 0); - - ASSERT_FALSE (store.account_get (transaction, account2, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, 0); - - ASSERT_FALSE (store.account_get (transaction, account3, stored_account_info)); - ASSERT_EQ (stored_account_info.confirmation_height, 0); + ASSERT_EQ (store.confirmation_height_count (transaction), 3); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height)); + ASSERT_EQ (confirmation_height, 0); + ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height)); + ASSERT_EQ (confirmation_height, 0); + ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height)); + ASSERT_EQ (confirmation_height, 0); } // Upgrade many accounts and check they all have a confirmation height of 0 (except genesis which should have 1) @@ -1715,18 +1726,11 @@ TEST (block_store, upgrade_confirmation_height_many) nano::mdb_store store (error, logger, path); auto transaction (store.tx_begin_read ()); ASSERT_EQ (store.account_count (transaction), total_num_accounts); + ASSERT_EQ (store.confirmation_height_count (transaction), total_num_accounts); - for (auto i (store.latest_begin (transaction)), n (store.latest_end ()); i != n; ++i) + for (auto i (store.confirmation_height_begin (transaction)), n (store.confirmation_height_end ()); i != n; ++i) { - nano::account_info & current (i->second); - if (i->first == nano::genesis_account) - { - ASSERT_EQ (current.confirmation_height, 1); - } - else - { - ASSERT_EQ (current.confirmation_height, 0); - } + ASSERT_EQ (i->second, (i->first == nano::genesis_account) ? 1 : 0); } } diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index cad65f1d11..4b1e24eac8 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -56,7 +56,9 @@ TEST (ledger, genesis_balance) ASSERT_GE (nano::seconds_since_epoch (), info.modified); ASSERT_LT (nano::seconds_since_epoch () - info.modified, 10); // Genesis block should be confirmed by default - ASSERT_EQ (info.confirmation_height, 1); + uint64_t confirmation_height; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (confirmation_height, 1); } // All nodes in the system should agree on the genesis balance @@ -2807,13 +2809,14 @@ TEST (ledger, confirmation_height_not_updated) ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, account_info)); nano::keypair key; nano::send_block send1 (account_info.head, key.pub, 50, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (account_info.head)); - ASSERT_EQ (1, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (1, confirmation_height); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); - ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_EQ (1, confirmation_height); nano::open_block open1 (send1.hash (), nano::genesis_account, key.pub, key.prv, key.pub, pool.generate (key.pub)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, open1).code); - nano::account_info account_info1; - ASSERT_FALSE (store->account_get (transaction, key.pub, account_info1)); - ASSERT_EQ (0, account_info1.confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, key.pub, confirmation_height)); + ASSERT_EQ (0, confirmation_height); } diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index 1db39d2bd6..a725ee0533 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -1518,13 +1518,13 @@ TEST (confirmation_height, single) auto send1 (std::make_shared (latest1, key1.pub, amount - system.nodes[0]->config.receive_minimum.number (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (latest1))); // Check confirmation heights before, should be uninitialized (1 for genesis). - nano::account_info account_info; + uint64_t confirmation_height; for (auto & node : system.nodes) { add_callback_stats (*node); auto transaction = node->store.tx_begin_read (); - ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (1, confirmation_height); } for (auto & node : system.nodes) @@ -1545,8 +1545,8 @@ TEST (confirmation_height, single) } auto transaction = node->store.tx_begin_read (); - ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (2, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (2, confirmation_height); // Rollbacks should fail as these blocks have been cemented ASSERT_TRUE (node->ledger.rollback (transaction, latest1)); @@ -1612,15 +1612,15 @@ TEST (confirmation_height, multiple_accounts) // Check confirmation heights of all the accounts are uninitialized (0), // as we have any just added them to the ledger and not processed any live transactions yet. - nano::account_info account_info; - ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); - ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_EQ (0, account_info.confirmation_height); - ASSERT_FALSE (node->store.account_get (transaction, key2.pub, account_info)); - ASSERT_EQ (0, account_info.confirmation_height); - ASSERT_FALSE (node->store.account_get (transaction, key3.pub, account_info)); - ASSERT_EQ (0, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (1, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); + ASSERT_EQ (0, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); + ASSERT_EQ (0, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height)); + ASSERT_EQ (0, confirmation_height); } // The nodes process a live receive which propagates across to all accounts @@ -1644,19 +1644,24 @@ TEST (confirmation_height, multiple_accounts) } nano::account_info account_info; + uint64_t confirmation_height; auto & store = node->store; auto transaction = node->store.tx_begin_read (); ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (4, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (4, confirmation_height); ASSERT_EQ (4, account_info.block_count); ASSERT_FALSE (store.account_get (transaction, key1.pub, account_info)); - ASSERT_EQ (2, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); + ASSERT_EQ (2, confirmation_height); ASSERT_EQ (3, account_info.block_count); ASSERT_FALSE (store.account_get (transaction, key2.pub, account_info)); - ASSERT_EQ (3, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); + ASSERT_EQ (3, confirmation_height); ASSERT_EQ (4, account_info.block_count); ASSERT_FALSE (store.account_get (transaction, key3.pub, account_info)); - ASSERT_EQ (2, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height)); + ASSERT_EQ (2, confirmation_height); ASSERT_EQ (2, account_info.block_count); // The accounts for key1 and key2 have 1 more block in the chain than is confirmed. @@ -1724,9 +1729,9 @@ TEST (confirmation_height, gap_bootstrap) auto unchecked_count (node1.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 2); - nano::account_info account_info; - ASSERT_FALSE (node1.store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (1, confirmation_height); } // Now complete the chain where the block comes in on the bootstrap network. @@ -1739,11 +1744,11 @@ TEST (confirmation_height, gap_bootstrap) auto unchecked_count (node1.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 0); - nano::account_info account_info; - ASSERT_FALSE (node1.store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); - ASSERT_FALSE (node1.store.account_get (transaction, destination.pub, account_info)); - ASSERT_EQ (0, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (1, confirmation_height); + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, destination.pub, confirmation_height)); + ASSERT_EQ (0, confirmation_height); } ASSERT_EQ (0, node1.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (0, node1.stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); @@ -1792,9 +1797,9 @@ TEST (confirmation_height, gap_live) // Confirmation heights should not be updated { auto transaction = node->store.tx_begin_read (); - nano::account_info account_info; - ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (1, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (1, confirmation_height); } // Now complete the chain where the block comes in on the live network @@ -1818,11 +1823,11 @@ TEST (confirmation_height, gap_live) auto unchecked_count (node->store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 0); - nano::account_info account_info; - ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (4, account_info.confirmation_height); - ASSERT_FALSE (node->store.account_get (transaction, destination.pub, account_info)); - ASSERT_EQ (3, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (4, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, destination.pub, confirmation_height)); + ASSERT_EQ (3, confirmation_height); ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (6, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); @@ -1894,12 +1899,15 @@ TEST (confirmation_height, send_receive_between_2_accounts) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; + uint64_t confirmation_height; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (6, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (6, confirmation_height); ASSERT_EQ (7, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_EQ (5, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); + ASSERT_EQ (5, confirmation_height); ASSERT_EQ (5, account_info.block_count); ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); @@ -1957,7 +1965,9 @@ TEST (confirmation_height, send_receive_self) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (7, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (7, confirmation_height); ASSERT_EQ (8, account_info.block_count); ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (6, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); @@ -2050,16 +2060,20 @@ TEST (confirmation_height, all_block_types) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; + uint64_t confirmation_height; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (3, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (3, confirmation_height); ASSERT_LE (4, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_EQ (6, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); + ASSERT_EQ (6, confirmation_height); ASSERT_LE (7, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key2.pub, account_info)); - ASSERT_EQ (7, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); + ASSERT_EQ (7, confirmation_height); ASSERT_LE (8, account_info.block_count); ASSERT_EQ (15, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); @@ -2107,20 +2121,12 @@ TEST (confirmation_height, conflict_rollback_cemented) { auto transaction (system.nodes[0]->store.tx_begin_write ()); ASSERT_TRUE (node1.store.block_exists (transaction, publish1.block->hash ())); - - nano::account_info info; - node1.store.account_get (transaction, nano::genesis_account, info); - info.confirmation_height = 2; - node1.store.account_put (transaction, nano::genesis_account, info); + node1.store.confirmation_height_put (transaction, nano::genesis_account, 2); } { auto transaction (system.nodes[1]->store.tx_begin_write ()); ASSERT_TRUE (node2.store.block_exists (transaction, publish2.block->hash ())); - - nano::account_info info; - node2.store.account_get (transaction, nano::genesis_account, info); - info.confirmation_height = 2; - node1.store.account_put (transaction, nano::genesis_account, info); + node2.store.confirmation_height_put (transaction, nano::genesis_account, 2); } auto rollback_log_entry = boost::str (boost::format ("Failed to roll back %1%") % send2->hash ().to_string ()); diff --git a/nano/core_test/versioning.cpp b/nano/core_test/versioning.cpp index 3591f1ff6e..beaa8d0351 100644 --- a/nano/core_test/versioning.cpp +++ b/nano/core_test/versioning.cpp @@ -37,7 +37,9 @@ TEST (versioning, account_info_v1) ASSERT_EQ (v1.modified, v_latest.modified); ASSERT_EQ (v1.rep_block, v_latest.rep_block); ASSERT_EQ (1, v_latest.block_count); - ASSERT_EQ (0, v_latest.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); + ASSERT_EQ (0, confirmation_height); ASSERT_EQ (nano::epoch::epoch_0, v_latest.epoch); } @@ -73,7 +75,9 @@ TEST (versioning, account_info_v5) ASSERT_EQ (v5.modified, v_latest.modified); ASSERT_EQ (v5.rep_block, v_latest.rep_block); ASSERT_EQ (1, v_latest.block_count); - ASSERT_EQ (0, v_latest.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); + ASSERT_EQ (0, confirmation_height); ASSERT_EQ (nano::epoch::epoch_0, v_latest.epoch); } @@ -109,6 +113,8 @@ TEST (versioning, account_info_v13) ASSERT_EQ (v13.modified, v_latest.modified); ASSERT_EQ (v13.rep_block, v_latest.rep_block); ASSERT_EQ (v13.block_count, v_latest.block_count); - ASSERT_EQ (0, v_latest.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); + ASSERT_EQ (0, confirmation_height); ASSERT_EQ (v13.epoch, v_latest.epoch); } diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index c47942b5e1..10b896ff03 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -823,10 +823,12 @@ int main (int argc, char * const * argv) } nano::account_info const & info (i->second); nano::account const & account (i->first); + uint64_t confirmation_height; + node.node->store.confirmation_height_get (transaction, account, confirmation_height); - if (info.confirmation_height > info.block_count) + if (confirmation_height > info.block_count) { - std::cerr << "Confirmation height " << info.confirmation_height << " greater than block count " << info.block_count << " for account: " << account.to_account () << std::endl; + std::cerr << "Confirmation height " << confirmation_height << " greater than block count " << info.block_count << " for account: " << account.to_account () << std::endl; } auto hash (info.open_block); diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index 60210882fc..1a1ccb7f10 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -72,8 +72,11 @@ void nano::active_transactions::confirm_frontiers (nano::transaction const & tra nano::account_info info; auto error = node.store.account_get (transaction_a, cementable_account.account, info); release_assert (!error); + uint64_t confirmation_height; + error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height); + release_assert (!error); - if (info.block_count > info.confirmation_height && !node.pending_confirmation_height.is_processing_block (info.head)) + if (info.block_count > confirmation_height && !node.pending_confirmation_height.is_processing_block (info.head)) { auto block (node.store.block_get (transaction_a, info.head)); if (!start (block)) @@ -362,9 +365,13 @@ void nano::active_transactions::prioritize_frontiers_for_confirmation (nano::tra { auto const & account (i->first); auto const & info (i->second); - if (info.block_count > info.confirmation_height && !node.pending_confirmation_height.is_processing_block (info.head)) + + uint64_t confirmation_height; + release_assert (!node.store.confirmation_height_get (transaction_a, account, confirmation_height)); + + if (info.block_count > confirmation_height && !node.pending_confirmation_height.is_processing_block (info.head)) { - auto num_uncemented = info.block_count - info.confirmation_height; + auto num_uncemented = info.block_count - confirmation_height; lk.lock (); auto it = priority_cementable_frontiers.find (account); if (it != priority_cementable_frontiers.end ()) diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index df43129309..ac5d6d0908 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -310,21 +310,20 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map nano::account account; if (!account.decode_account (account_str)) { - nano::account_info account_info; + uint64_t confirmation_height; auto transaction (node.node->store.tx_begin_read ()); - if (!node.node->store.account_get (transaction, account, account_info)) + if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height)) { auto transaction (node.node->store.tx_begin_write ()); auto conf_height_reset_num = 0; if (account == node.node->network_params.ledger.genesis_account) { conf_height_reset_num = 1; - account_info.confirmation_height = conf_height_reset_num; - node.node->store.account_put (transaction, account, account_info); + node.node->store.confirmation_height_put (transaction, account, confirmation_height); } else { - node.node->store.confirmation_height_clear (transaction, account, account_info); + node.node->store.confirmation_height_clear (transaction, account, confirmation_height); } std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl; @@ -963,11 +962,6 @@ void reset_confirmation_heights (nano::block_store & store) // Then make sure the confirmation height of the genesis account open block is 1 nano::network_params network_params; - auto const & genesis_account = network_params.ledger.genesis_account; - nano::account_info account_info; - auto error = store.account_get (transaction, genesis_account, account_info); - release_assert (!error); - account_info.confirmation_height = 1; - store.account_put (transaction, genesis_account, account_info); + store.confirmation_height_put (transaction, network_params.ledger.genesis_account, 1); } } diff --git a/nano/node/confirmation_height_processor.cpp b/nano/node/confirmation_height_processor.cpp index b5d4e1b5e9..cedfb7fa22 100644 --- a/nano/node/confirmation_height_processor.cpp +++ b/nano/node/confirmation_height_processor.cpp @@ -103,7 +103,6 @@ void nano::confirmation_height_processor::add_confirmation_height (nano::block_h { boost::optional receive_details; auto current = hash_a; - nano::account_info account_info; assert (receive_source_pairs_size == 0); release_assert (receive_source_pairs.empty ()); @@ -130,8 +129,8 @@ void nano::confirmation_height_processor::add_confirmation_height (nano::block_h auto block_height (store.block_account_height (read_transaction, current)); nano::account account (store.block_account (read_transaction, current)); - release_assert (!store.account_get (read_transaction, account, account_info)); - auto confirmation_height = account_info.confirmation_height; + uint64_t confirmation_height; + release_assert (!store.confirmation_height_get (read_transaction, account, confirmation_height)); auto iterated_height = confirmation_height; auto account_it = confirmed_iterated_pairs.find (account); if (account_it != confirmed_iterated_pairs.cend ()) @@ -262,7 +261,6 @@ void nano::confirmation_height_processor::add_confirmation_height (nano::block_h */ bool nano::confirmation_height_processor::write_pending (std::deque & all_pending_a) { - nano::account_info account_info; auto total_pending_write_block_count = std::accumulate (all_pending_a.cbegin (), all_pending_a.cend (), uint64_t (0), [](uint64_t total, conf_height_details const & conf_height_details_a) { return total += conf_height_details_a.num_blocks_confirmed; }); @@ -275,9 +273,10 @@ bool nano::confirmation_height_processor::write_pending (std::deque account_info.confirmation_height) + if (pending.height > confirmation_height) { #ifndef NDEBUG // Do more thorough checking in Debug mode, indicates programming error. @@ -300,10 +299,10 @@ bool nano::confirmation_height_processor::write_pending (std::deque ("pending", false); auto transaction (node.store.tx_begin_read ()); nano::account_info info; - if (!node.store.account_get (transaction, account, info)) + uint64_t confirmation_height; + auto error = node.store.account_get (transaction, account, info) | node.store.confirmation_height_get (transaction, account, confirmation_height); + if (!error) { response_l.put ("frontier", info.head.to_string ()); response_l.put ("open_block", info.open_block.to_string ()); @@ -492,7 +494,7 @@ void nano::json_handler::account_info () response_l.put ("modified_timestamp", std::to_string (info.modified)); response_l.put ("block_count", std::to_string (info.block_count)); response_l.put ("account_version", info.epoch == nano::epoch::epoch_1 ? "1" : "0"); - response_l.put ("confirmation_height", std::to_string (info.confirmation_height)); + response_l.put ("confirmation_height", std::to_string (confirmation_height)); if (representative) { auto block (node.store.block_get (transaction, info.rep_block)); diff --git a/nano/node/lmdb.cpp b/nano/node/lmdb.cpp deleted file mode 100644 index 13b6fc26da..0000000000 --- a/nano/node/lmdb.cpp +++ /dev/null @@ -1,1755 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace nano -{ -template <> -void * mdb_val::data () const -{ - return value.mv_data; -} - -template <> -size_t mdb_val::size () const -{ - return value.mv_size; -} - -template <> -mdb_val::db_val (nano::DB_val const & value_a, nano::epoch epoch_a) : -value ({ value_a.size, value_a.data }), -epoch (epoch_a) -{ -} -} - -nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a) -{ - boost::system::error_code error_mkdir, error_chmod; - if (path_a.has_parent_path ()) - { - boost::filesystem::create_directories (path_a.parent_path (), error_mkdir); - nano::set_secure_perm_directory (path_a.parent_path (), error_chmod); - if (!error_mkdir) - { - auto status1 (mdb_env_create (&environment)); - release_assert (status1 == 0); - auto status2 (mdb_env_set_maxdbs (environment, max_dbs_a)); - release_assert (status2 == 0); - auto map_size = map_size_a; - auto max_valgrind_map_size = 16 * 1024 * 1024; - if (running_within_valgrind () && map_size_a > max_valgrind_map_size) - { - // In order to run LMDB under Valgrind, the maximum map size must be smaller than half your available RAM - map_size = max_valgrind_map_size; - } - auto status3 (mdb_env_set_mapsize (environment, map_size)); - release_assert (status3 == 0); - // It seems if there's ever more threads than mdb_env_set_maxreaders has read slots available, we get failures on transaction creation unless MDB_NOTLS is specified - // This can happen if something like 256 io_threads are specified in the node config - // MDB_NORDAHEAD will allow platforms that support it to load the DB in memory as needed. - // MDB_NOMEMINIT prevents zeroing malloc'ed pages. Can provide improvement for non-sensitive data but may make memory checkers noisy (e.g valgrind). - auto environment_flags = MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD; - if (!running_within_valgrind () && use_no_mem_init_a) - { - environment_flags |= MDB_NOMEMINIT; - } - auto status4 (mdb_env_open (environment, path_a.string ().c_str (), environment_flags, 00600)); - if (status4 != 0) - { - std::cerr << "Could not open lmdb environment: " << status4; - char * error_str (mdb_strerror (status4)); - if (error_str) - { - std::cerr << ", " << error_str; - } - std::cerr << std::endl; - } - release_assert (status4 == 0); - error_a = status4 != 0; - } - else - { - error_a = true; - environment = nullptr; - } - } - else - { - error_a = true; - environment = nullptr; - } -} - -nano::mdb_env::~mdb_env () -{ - if (environment != nullptr) - { - mdb_env_close (environment); - } -} - -nano::mdb_env::operator MDB_env * () const -{ - return environment; -} - -nano::read_transaction nano::mdb_env::tx_begin_read (mdb_txn_callbacks mdb_txn_callbacks) const -{ - return nano::read_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; -} - -nano::write_transaction nano::mdb_env::tx_begin_write (mdb_txn_callbacks mdb_txn_callbacks) const -{ - return nano::write_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; -} - -MDB_txn * nano::mdb_env::tx (nano::transaction const & transaction_a) const -{ - return static_cast (transaction_a.get_handle ()); -} - -nano::read_mdb_txn::read_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) : -txn_callbacks (txn_callbacks_a) -{ - auto status (mdb_txn_begin (environment_a, nullptr, MDB_RDONLY, &handle)); - release_assert (status == 0); - txn_callbacks.txn_start (this); -} - -nano::read_mdb_txn::~read_mdb_txn () -{ - // This uses commit rather than abort, as it is needed when opening databases with a read only transaction - auto status (mdb_txn_commit (handle)); - release_assert (status == MDB_SUCCESS); - txn_callbacks.txn_end (this); -} - -void nano::read_mdb_txn::reset () const -{ - mdb_txn_reset (handle); - txn_callbacks.txn_end (this); -} - -void nano::read_mdb_txn::renew () const -{ - auto status (mdb_txn_renew (handle)); - release_assert (status == 0); - txn_callbacks.txn_start (this); -} - -void * nano::read_mdb_txn::get_handle () const -{ - return handle; -} - -nano::write_mdb_txn::write_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) : -env (environment_a), -txn_callbacks (txn_callbacks_a) -{ - renew (); -} - -nano::write_mdb_txn::~write_mdb_txn () -{ - commit (); -} - -void nano::write_mdb_txn::commit () const -{ - auto status (mdb_txn_commit (handle)); - release_assert (status == MDB_SUCCESS); - txn_callbacks.txn_end (this); -} - -void nano::write_mdb_txn::renew () -{ - auto status (mdb_txn_begin (env, nullptr, 0, &handle)); - release_assert (status == MDB_SUCCESS); - txn_callbacks.txn_start (this); -} - -void * nano::write_mdb_txn::get_handle () const -{ - return handle; -} - -template -nano::mdb_iterator::mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, nano::epoch epoch_a) : -cursor (nullptr) -{ - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); - release_assert (status == 0); - auto status2 (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_FIRST)); - release_assert (status2 == 0 || status2 == MDB_NOTFOUND); - if (status2 != MDB_NOTFOUND) - { - auto status3 (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_GET_CURRENT)); - release_assert (status3 == 0 || status3 == MDB_NOTFOUND); - if (current.first.size () != sizeof (T)) - { - clear (); - } - } - else - { - clear (); - } -} - -template -nano::mdb_iterator::mdb_iterator (std::nullptr_t, nano::epoch epoch_a) : -cursor (nullptr) -{ - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; -} - -template -nano::mdb_iterator::mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a, nano::epoch epoch_a) : -cursor (nullptr) -{ - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); - release_assert (status == 0); - current.first = val_a; - auto status2 (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_SET_RANGE)); - release_assert (status2 == 0 || status2 == MDB_NOTFOUND); - if (status2 != MDB_NOTFOUND) - { - auto status3 (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_GET_CURRENT)); - release_assert (status3 == 0 || status3 == MDB_NOTFOUND); - if (current.first.size () != sizeof (T)) - { - clear (); - } - } - else - { - clear (); - } -} - -template -nano::mdb_iterator::mdb_iterator (nano::mdb_iterator && other_a) -{ - cursor = other_a.cursor; - other_a.cursor = nullptr; - current = other_a.current; -} - -template -nano::mdb_iterator::~mdb_iterator () -{ - if (cursor != nullptr) - { - mdb_cursor_close (cursor); - } -} - -template -nano::store_iterator_impl & nano::mdb_iterator::operator++ () -{ - assert (cursor != nullptr); - auto status (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_NEXT)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status == MDB_NOTFOUND) - { - clear (); - } - if (current.first.size () != sizeof (T)) - { - clear (); - } - return *this; -} - -template -nano::mdb_iterator & nano::mdb_iterator::operator= (nano::mdb_iterator && other_a) -{ - if (cursor != nullptr) - { - mdb_cursor_close (cursor); - } - cursor = other_a.cursor; - other_a.cursor = nullptr; - current = other_a.current; - other_a.clear (); - return *this; -} - -template -std::pair * nano::mdb_iterator::operator-> () -{ - return ¤t; -} - -template -bool nano::mdb_iterator::operator== (nano::store_iterator_impl const & base_a) const -{ - auto const other_a (boost::polymorphic_downcast const *> (&base_a)); - auto result (current.first.data () == other_a->current.first.data ()); - assert (!result || (current.first.size () == other_a->current.first.size ())); - assert (!result || (current.second.data () == other_a->current.second.data ())); - assert (!result || (current.second.size () == other_a->current.second.size ())); - return result; -} - -template -void nano::mdb_iterator::clear () -{ - current.first = nano::mdb_val (current.first.epoch); - current.second = nano::mdb_val (current.second.epoch); - assert (is_end_sentinal ()); -} - -template -MDB_txn * nano::mdb_iterator::tx (nano::transaction const & transaction_a) const -{ - return static_cast (transaction_a.get_handle ()); -} - -template -bool nano::mdb_iterator::is_end_sentinal () const -{ - return current.first.size () == 0; -} - -template -void nano::mdb_iterator::fill (std::pair & value_a) const -{ - if (current.first.size () != 0) - { - value_a.first = static_cast (current.first); - } - else - { - value_a.first = T (); - } - if (current.second.size () != 0) - { - value_a.second = static_cast (current.second); - } - else - { - value_a.second = U (); - } -} - -template -std::pair * nano::mdb_merge_iterator::operator-> () -{ - return least_iterator ().operator-> (); -} - -template -nano::mdb_merge_iterator::mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a) : -impl1 (std::make_unique> (transaction_a, db1_a, nano::epoch::epoch_0)), -impl2 (std::make_unique> (transaction_a, db2_a, nano::epoch::epoch_1)) -{ -} - -template -nano::mdb_merge_iterator::mdb_merge_iterator (std::nullptr_t) : -impl1 (std::make_unique> (nullptr, nano::epoch::epoch_0)), -impl2 (std::make_unique> (nullptr, nano::epoch::epoch_1)) -{ -} - -template -nano::mdb_merge_iterator::mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a, MDB_val const & val_a) : -impl1 (std::make_unique> (transaction_a, db1_a, val_a, nano::epoch::epoch_0)), -impl2 (std::make_unique> (transaction_a, db2_a, val_a, nano::epoch::epoch_1)) -{ -} - -template -nano::mdb_merge_iterator::mdb_merge_iterator (nano::mdb_merge_iterator && other_a) -{ - impl1 = std::move (other_a.impl1); - impl2 = std::move (other_a.impl2); -} - -template -nano::mdb_merge_iterator::~mdb_merge_iterator () -{ -} - -template -nano::store_iterator_impl & nano::mdb_merge_iterator::operator++ () -{ - ++least_iterator (); - return *this; -} - -template -bool nano::mdb_merge_iterator::is_end_sentinal () const -{ - return least_iterator ().is_end_sentinal (); -} - -template -void nano::mdb_merge_iterator::fill (std::pair & value_a) const -{ - auto & current (least_iterator ()); - if (current->first.size () != 0) - { - value_a.first = static_cast (current->first); - } - else - { - value_a.first = T (); - } - if (current->second.size () != 0) - { - value_a.second = static_cast (current->second); - } - else - { - value_a.second = U (); - } -} - -template -bool nano::mdb_merge_iterator::operator== (nano::store_iterator_impl const & base_a) const -{ - assert ((dynamic_cast const *> (&base_a) != nullptr) && "Incompatible iterator comparison"); - auto & other (static_cast const &> (base_a)); - return *impl1 == *other.impl1 && *impl2 == *other.impl2; -} - -template -nano::mdb_iterator & nano::mdb_merge_iterator::least_iterator () const -{ - nano::mdb_iterator * result; - if (impl1->is_end_sentinal ()) - { - result = impl2.get (); - } - else if (impl2->is_end_sentinal ()) - { - result = impl1.get (); - } - else - { - auto key_cmp (mdb_cmp (mdb_cursor_txn (impl1->cursor), mdb_cursor_dbi (impl1->cursor), impl1->current.first, impl2->current.first)); - - if (key_cmp < 0) - { - result = impl1.get (); - } - else if (key_cmp > 0) - { - result = impl2.get (); - } - else - { - auto val_cmp (mdb_cmp (mdb_cursor_txn (impl1->cursor), mdb_cursor_dbi (impl1->cursor), impl1->current.second, impl2->current.second)); - result = val_cmp < 0 ? impl1.get () : impl2.get (); - } - } - return *result; -} - -nano::wallet_value::wallet_value (nano::mdb_val const & val_a) -{ - assert (val_a.size () == sizeof (*this)); - std::copy (reinterpret_cast (val_a.data ()), reinterpret_cast (val_a.data ()) + sizeof (key), key.chars.begin ()); - std::copy (reinterpret_cast (val_a.data ()) + sizeof (key), reinterpret_cast (val_a.data ()) + sizeof (key) + sizeof (work), reinterpret_cast (&work)); -} - -nano::wallet_value::wallet_value (nano::uint256_union const & key_a, uint64_t work_a) : -key (key_a), -work (work_a) -{ -} - -nano::mdb_val nano::wallet_value::val () const -{ - static_assert (sizeof (*this) == sizeof (key) + sizeof (work), "Class not packed"); - return nano::mdb_val (sizeof (*this), const_cast (this)); -} - -template class nano::mdb_iterator; -template class nano::mdb_iterator; -template class nano::mdb_iterator; -template class nano::mdb_iterator; -template class nano::mdb_iterator>; -template class nano::mdb_iterator>; -template class nano::mdb_iterator; -template class nano::mdb_iterator, nano::no_value>; - -nano::store_iterator nano::mdb_store::representation_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, representation)); - return result; -} - -nano::store_iterator nano::mdb_store::representation_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator nano::mdb_store::unchecked_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, unchecked)); - return result; -} - -nano::store_iterator nano::mdb_store::unchecked_begin (nano::transaction const & transaction_a, nano::unchecked_key const & key_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, unchecked, nano::mdb_val (key_a))); - return result; -} - -nano::store_iterator nano::mdb_store::unchecked_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator> nano::mdb_store::vote_begin (nano::transaction const & transaction_a) -{ - return nano::store_iterator> (std::make_unique>> (transaction_a, vote)); -} - -nano::store_iterator> nano::mdb_store::vote_end () -{ - return nano::store_iterator> (nullptr); -} - -nano::mdb_store::mdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) : -logger (logger_a), -env (error_a, path_a, lmdb_max_dbs, true), -mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), -txn_tracking_enabled (txn_tracking_config_a.enable) -{ - if (!error_a) - { - auto is_fully_upgraded (false); - { - auto transaction (tx_begin_read ()); - auto err = mdb_dbi_open (env.tx (transaction), "meta", 0, &meta); - if (err == MDB_SUCCESS) - { - is_fully_upgraded = (version_get (transaction) == version); - mdb_dbi_close (env, meta); - } - } - - // Only open a write lock when upgrades are needed. This is because CLI commands - // open inactive nodes which can otherwise be locked here if there is a long write - // (can be a few minutes with the --fastbootstrap flag for instance) - if (!is_fully_upgraded) - { - auto transaction (tx_begin_write ()); - open_databases (error_a, transaction, MDB_CREATE); - if (!error_a) - { - error_a |= do_upgrades (transaction, batch_size); - } - } - else - { - auto transaction (tx_begin_read ()); - open_databases (error_a, transaction, 0); - } - - if (!error_a && drop_unchecked) - { - auto transaction (tx_begin_write ()); - unchecked_clear (transaction); - } - } -} - -void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time) -{ - mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time); -} - -nano::write_transaction nano::mdb_store::tx_begin_write () -{ - return env.tx_begin_write (create_txn_callbacks ()); -} - -nano::read_transaction nano::mdb_store::tx_begin_read () -{ - return env.tx_begin_read (create_txn_callbacks ()); -} - -nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () -{ - nano::mdb_txn_callbacks mdb_txn_callbacks; - if (txn_tracking_enabled) - { - // clang-format off - mdb_txn_callbacks.txn_start = ([&mdb_txn_tracker = mdb_txn_tracker](const nano::transaction_impl * transaction_impl) { - mdb_txn_tracker.add (transaction_impl); - }); - mdb_txn_callbacks.txn_end = ([&mdb_txn_tracker = mdb_txn_tracker](const nano::transaction_impl * transaction_impl) { - mdb_txn_tracker.erase (transaction_impl); - }); - // clang-format on - } - return mdb_txn_callbacks; -} - -void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) -{ - error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts_v1", flags, &accounts_v1) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &representation) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0; - if (!full_sideband (transaction_a)) - { - error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks_info", flags, &blocks_info) != 0; - } -} - -void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a) -{ - nano::uint256_union version_key (1); - nano::uint256_union version_value (version_a); - auto status (mdb_put (env.tx (transaction_a), meta, nano::mdb_val (version_key), nano::mdb_val (version_value), 0)); - release_assert (status == 0); - if (blocks_info == 0 && !full_sideband (transaction_a)) - { - auto status (mdb_dbi_open (env.tx (transaction_a), "blocks_info", MDB_CREATE, &blocks_info)); - release_assert (status == MDB_SUCCESS); - } - if (blocks_info != 0 && full_sideband (transaction_a)) - { - auto status (mdb_drop (env.tx (transaction_a), blocks_info, 1)); - release_assert (status == MDB_SUCCESS); - blocks_info = 0; - } -} - -int nano::mdb_store::version_get (nano::transaction const & transaction_a) const -{ - nano::uint256_union version_key (1); - nano::mdb_val data; - auto error (mdb_get (env.tx (transaction_a), meta, nano::mdb_val (version_key), data)); - int result (1); - if (error != MDB_NOTFOUND) - { - nano::uint256_union version_value (data); - assert (version_value.qwords[2] == 0 && version_value.qwords[1] == 0 && version_value.qwords[0] == 0); - result = version_value.number ().convert_to (); - } - return result; -} - -void nano::mdb_store::peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) -{ - nano::mdb_val zero (static_cast (0)); - auto status (mdb_put (env.tx (transaction_a), peers, nano::mdb_val (endpoint_a), zero, 0)); - release_assert (status == 0); -} - -void nano::mdb_store::peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) -{ - auto status (mdb_del (env.tx (transaction_a), peers, nano::mdb_val (endpoint_a), nullptr)); - release_assert (status == 0); -} - -bool nano::mdb_store::peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const -{ - nano::mdb_val junk; - auto status (mdb_get (env.tx (transaction_a), peers, nano::mdb_val (endpoint_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - return (status == 0); -} - -size_t nano::mdb_store::peer_count (nano::transaction const & transaction_a) const -{ - return count (transaction_a, peers); -} - -void nano::mdb_store::peer_clear (nano::transaction const & transaction_a) -{ - auto status (mdb_drop (env.tx (transaction_a), peers, 0)); - release_assert (status == 0); -} - -nano::store_iterator nano::mdb_store::peers_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, peers)); - return result; -} - -nano::store_iterator nano::mdb_store::peers_end () -{ - nano::store_iterator result (nano::store_iterator (nullptr)); - return result; -} - -bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size_t batch_size) -{ - auto error (false); - auto version_l = version_get (transaction_a); - switch (version_l) - { - case 1: - upgrade_v1_to_v2 (transaction_a); - case 2: - upgrade_v2_to_v3 (transaction_a); - case 3: - upgrade_v3_to_v4 (transaction_a); - case 4: - upgrade_v4_to_v5 (transaction_a); - case 5: - upgrade_v5_to_v6 (transaction_a); - case 6: - upgrade_v6_to_v7 (transaction_a); - case 7: - upgrade_v7_to_v8 (transaction_a); - case 8: - upgrade_v8_to_v9 (transaction_a); - case 9: - upgrade_v9_to_v10 (transaction_a); - case 10: - upgrade_v10_to_v11 (transaction_a); - case 11: - upgrade_v11_to_v12 (transaction_a); - case 12: - upgrade_v12_to_v13 (transaction_a, batch_size); - case 13: - upgrade_v13_to_v14 (transaction_a); - case 14: - break; - default: - logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); - error = true; - break; - } - return error; -} - -void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 2); - nano::account account (1); - while (!account.is_zero ()) - { - nano::mdb_iterator i (transaction_a, accounts_v0, nano::mdb_val (account)); - std::cerr << std::hex; - if (i != nano::mdb_iterator (nullptr)) - { - account = nano::uint256_union (i->first); - nano::account_info_v1 v1 (i->second); - nano::account_info_v5 v2; - v2.balance = v1.balance; - v2.head = v1.head; - v2.modified = v1.modified; - v2.rep_block = v1.rep_block; - auto block (block_get (transaction_a, v1.head)); - while (!block->previous ().is_zero ()) - { - block = block_get (transaction_a, block->previous ()); - } - v2.open_block = block->hash (); - auto status (mdb_put (env.tx (transaction_a), accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v2), &v2), 0)); - release_assert (status == 0); - account = account.number () + 1; - } - else - { - account.clear (); - } - } -} - -void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 3); - mdb_drop (env.tx (transaction_a), representation, 0); - for (auto i (std::make_unique> (transaction_a, accounts_v0)), n (std::make_unique> (nullptr)); *i != *n; ++(*i)) - { - nano::account account_l ((*i)->first); - nano::account_info_v5 info ((*i)->second); - representative_visitor visitor (transaction_a, *this); - visitor.compute (info.head); - assert (!visitor.result.is_zero ()); - info.rep_block = visitor.result; - auto impl (boost::polymorphic_downcast *> (i.get ())); - mdb_cursor_put (impl->cursor, nano::mdb_val (account_l), nano::mdb_val (sizeof (info), &info), MDB_CURRENT); - representation_add (transaction_a, visitor.result, info.balance.number ()); - } -} - -void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 4); - std::queue> items; - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, pending_v0))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - nano::block_hash const & hash (i->first); - nano::pending_info_v3 const & info (i->second); - items.push (std::make_pair (nano::pending_key (info.destination, hash), nano::pending_info (info.source, info.amount, nano::epoch::epoch_0))); - } - mdb_drop (env.tx (transaction_a), pending_v0, 0); - while (!items.empty ()) - { - pending_put (transaction_a, items.front ().first, items.front ().second); - items.pop (); - } -} - -void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 5); - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, accounts_v0))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - nano::account_info_v5 const & info (i->second); - nano::block_hash successor (0); - auto block (block_get (transaction_a, info.head)); - while (block != nullptr) - { - auto hash (block->hash ()); - if (block_successor (transaction_a, hash).is_zero () && !successor.is_zero ()) - { - std::vector vector; - { - nano::vectorstream stream (vector); - block->serialize (stream); - nano::write (stream, successor.bytes); - } - block_raw_put (transaction_a, vector, block->type (), nano::epoch::epoch_0, hash); - if (!block->previous ().is_zero ()) - { - nano::block_type type; - auto value (block_raw_get (transaction_a, block->previous (), type)); - auto version (block_version (transaction_a, block->previous ())); - assert (value.size () != 0); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.end () - nano::block_sideband::size (type)); - block_raw_put (transaction_a, data, type, version, block->previous ()); - } - } - successor = hash; - block = block_get (transaction_a, block->previous ()); - } - } -} - -void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 6); - std::deque> headers; - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, accounts_v0))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - nano::account const & account (i->first); - nano::account_info_v5 info_old (i->second); - uint64_t block_count (0); - auto hash (info_old.head); - while (!hash.is_zero ()) - { - ++block_count; - auto block (block_get (transaction_a, hash)); - assert (block != nullptr); - hash = block->previous (); - } - headers.emplace_back (account, nano::account_info_v13{ info_old.head, info_old.rep_block, info_old.open_block, info_old.balance, info_old.modified, block_count, nano::epoch::epoch_0 }); - } - for (auto i (headers.begin ()), n (headers.end ()); i != n; ++i) - { - auto status (mdb_put (env.tx (transaction_a), accounts_v0, nano::mdb_val (i->first), nano::mdb_val (i->second), 0)); - release_assert (status == 0); - } -} - -void nano::mdb_store::upgrade_v6_to_v7 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 7); - mdb_drop (env.tx (transaction_a), unchecked, 0); -} - -void nano::mdb_store::upgrade_v7_to_v8 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 8); - mdb_drop (env.tx (transaction_a), unchecked, 1); - mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE | MDB_DUPSORT, &unchecked); -} - -void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 9); - MDB_dbi sequence; - mdb_dbi_open (env.tx (transaction_a), "sequence", MDB_CREATE | MDB_DUPSORT, &sequence); - nano::genesis genesis; - std::shared_ptr block (std::move (genesis.open)); - nano::keypair junk; - for (nano::mdb_iterator i (transaction_a, sequence), n (nano::mdb_iterator (nullptr)); i != n; ++i) - { - nano::bufferstream stream (reinterpret_cast (i->second.data ()), i->second.size ()); - uint64_t sequence; - auto error (nano::try_read (stream, sequence)); - (void)error; - // Create a dummy vote with the same sequence number for easy upgrading. This won't have a valid signature. - nano::vote dummy (nano::account (i->first), junk.prv, sequence, block); - std::vector vector; - { - nano::vectorstream stream (vector); - dummy.serialize (stream); - } - auto status1 (mdb_put (env.tx (transaction_a), vote, nano::mdb_val (i->first), nano::mdb_val (vector.size (), vector.data ()), 0)); - release_assert (status1 == 0); - assert (!error); - } - mdb_drop (env.tx (transaction_a), sequence, 1); -} - -void nano::mdb_store::upgrade_v9_to_v10 (nano::transaction const & transaction_a) -{ -} - -void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 11); - MDB_dbi unsynced; - mdb_dbi_open (env.tx (transaction_a), "unsynced", MDB_CREATE | MDB_DUPSORT, &unsynced); - mdb_drop (env.tx (transaction_a), unsynced, 1); -} - -void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_a) -{ - version_put (transaction_a, 12); - mdb_drop (env.tx (transaction_a), unchecked, 1); - mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked); - MDB_dbi checksum; - mdb_dbi_open (env.tx (transaction_a), "checksum", MDB_CREATE, &checksum); - mdb_drop (env.tx (transaction_a), checksum, 1); -} - -void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_a, size_t const batch_size) -{ - size_t cost (0); - nano::account account (0); - auto const & not_an_account (network_params.random.not_an_account); - while (account != not_an_account) - { - nano::account first (0); - nano::account_info_v13 second; - { - nano::store_iterator current (std::make_unique> (transaction_a, accounts_v0, accounts_v1, nano::mdb_val (account))); - nano::store_iterator end (nullptr); - if (current != end) - { - first = current->first; - second = current->second; - } - } - if (!first.is_zero ()) - { - auto hash (second.open_block); - uint64_t height (1); - nano::block_sideband sideband; - while (!hash.is_zero ()) - { - if (cost >= batch_size) - { - logger.always_log (boost::str (boost::format ("Upgrading sideband information for account %1%... height %2%") % first.to_account ().substr (0, 24) % std::to_string (height))); - transaction_a.commit (); - std::this_thread::yield (); - transaction_a.renew (); - cost = 0; - } - auto block (block_get (transaction_a, hash, &sideband)); - assert (block != nullptr); - if (sideband.height == 0) - { - sideband.height = height; - block_put (transaction_a, hash, *block, sideband, block_version (transaction_a, hash)); - cost += 16; - } - else - { - cost += 1; - } - hash = sideband.successor; - ++height; - } - account = first.number () + 1; - } - else - { - account = not_an_account; - } - } - if (account == not_an_account) - { - logger.always_log ("Completed sideband upgrade"); - version_put (transaction_a, 13); - } -} - -void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_a) -{ - // Upgrade all accounts to have a confirmation of 0 (except genesis which should have 1) - version_put (transaction_a, 14); - nano::store_iterator i (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); - nano::store_iterator n (nullptr); - - std::vector> account_infos; - account_infos.reserve (account_count (transaction_a)); - for (; i != n; ++i) - { - nano::account_info_v13 const & account_info_v13 (i->second); - uint64_t confirmation_height = 0; - if (i->first == network_params.ledger.genesis_account) - { - confirmation_height = 1; - } - account_infos.emplace_back (i->first, nano::account_info{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, account_info_v13.epoch }); - } - - for (auto const & account_info : account_infos) - { - account_put (transaction_a, account_info.first, account_info.second); - } - - logger.always_log ("Completed confirmation height upgrade"); - - nano::uint256_union node_id_mdb_key (3); - auto error (mdb_del (env.tx (transaction_a), meta, nano::mdb_val (node_id_mdb_key), nullptr)); - release_assert (!error || error == MDB_NOTFOUND); -} - -void nano::mdb_store::clear (MDB_dbi db_a) -{ - auto transaction (tx_begin_write ()); - auto status (mdb_drop (env.tx (transaction), db_a, 0)); - release_assert (status == 0); -} - -nano::epoch nano::mdb_store::block_version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - nano::mdb_val value; - auto status (mdb_get (env.tx (transaction_a), state_blocks_v1, nano::mdb_val (hash_a), value)); - release_assert (status == 0 || status == MDB_NOTFOUND); - return status == 0 ? nano::epoch::epoch_1 : nano::epoch::epoch_0; -} - -MDB_dbi nano::mdb_store::block_database (nano::block_type type_a, nano::epoch epoch_a) -{ - if (type_a == nano::block_type::state) - { - assert (epoch_a == nano::epoch::epoch_0 || epoch_a == nano::epoch::epoch_1); - } - else - { - assert (epoch_a == nano::epoch::epoch_0); - } - MDB_dbi result = 0; - switch (type_a) - { - case nano::block_type::send: - result = send_blocks; - break; - case nano::block_type::receive: - result = receive_blocks; - break; - case nano::block_type::open: - result = open_blocks; - break; - case nano::block_type::change: - result = change_blocks; - break; - case nano::block_type::state: - switch (epoch_a) - { - case nano::epoch::epoch_0: - result = state_blocks_v0; - break; - case nano::epoch::epoch_1: - result = state_blocks_v1; - break; - default: - assert (false); - } - break; - default: - assert (false); - break; - } - return result; -} - -void nano::mdb_store::block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) -{ - MDB_dbi database_a = block_database (block_type_a, epoch_a); - MDB_val value{ data.size (), (void *)data.data () }; - auto status2 (mdb_put (env.tx (transaction_a), database_a, nano::mdb_val (hash_a), &value, 0)); - release_assert (status2 == 0); -} - -boost::optional nano::mdb_store::block_raw_get_by_type (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const -{ - nano::mdb_val value; - auto status (MDB_NOTFOUND); - switch (type_a) - { - case nano::block_type::send: - { - status = mdb_get (env.tx (transaction_a), send_blocks, nano::mdb_val (hash_a), value); - break; - } - case nano::block_type::receive: - { - status = mdb_get (env.tx (transaction_a), receive_blocks, nano::mdb_val (hash_a), value); - break; - } - case nano::block_type::open: - { - status = mdb_get (env.tx (transaction_a), open_blocks, nano::mdb_val (hash_a), value); - break; - } - case nano::block_type::change: - { - status = mdb_get (env.tx (transaction_a), change_blocks, nano::mdb_val (hash_a), value); - break; - } - case nano::block_type::state: - { - status = mdb_get (env.tx (transaction_a), state_blocks_v1, nano::mdb_val (hash_a), value); - if (status != 0) - { - status = mdb_get (env.tx (transaction_a), state_blocks_v0, nano::mdb_val (hash_a), value); - } - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - { - break; - } - } - - release_assert (status == MDB_SUCCESS || status == MDB_NOTFOUND); - boost::optional result; - if (status == MDB_SUCCESS) - { - result = nano::DB_val (value.size (), value.data ()); - } - - return result; -} - -template -std::shared_ptr nano::mdb_store::block_random (nano::transaction const & transaction_a, MDB_dbi database) -{ - nano::block_hash hash; - nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); - nano::store_iterator> existing (std::make_unique>> (transaction_a, database, nano::mdb_val (hash))); - if (existing == nano::store_iterator> (nullptr)) - { - existing = nano::store_iterator> (std::make_unique>> (transaction_a, database)); - } - auto end (nano::store_iterator> (nullptr)); - assert (existing != end); - return block_get (transaction_a, nano::block_hash (existing->first)); -} - -std::shared_ptr nano::mdb_store::block_random (nano::transaction const & transaction_a) -{ - auto count (block_count (transaction_a)); - release_assert (std::numeric_limits::max () > count.sum ()); - auto region = static_cast (nano::random_pool::generate_word32 (0, static_cast (count.sum () - 1))); - std::shared_ptr result; - if (region < count.send) - { - result = block_random (transaction_a, send_blocks); - } - else - { - region -= count.send; - if (region < count.receive) - { - result = block_random (transaction_a, receive_blocks); - } - else - { - region -= count.receive; - if (region < count.open) - { - result = block_random (transaction_a, open_blocks); - } - else - { - region -= count.open; - if (region < count.change) - { - result = block_random (transaction_a, change_blocks); - } - else - { - region -= count.change; - if (region < count.state_v0) - { - result = block_random (transaction_a, state_blocks_v0); - } - else - { - result = block_random (transaction_a, state_blocks_v1); - } - } - } - } - } - assert (result != nullptr); - return result; -} - -void nano::mdb_store::block_del (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status (mdb_del (env.tx (transaction_a), state_blocks_v1, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status != 0) - { - auto status (mdb_del (env.tx (transaction_a), state_blocks_v0, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status != 0) - { - auto status (mdb_del (env.tx (transaction_a), send_blocks, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status != 0) - { - auto status (mdb_del (env.tx (transaction_a), receive_blocks, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status != 0) - { - auto status (mdb_del (env.tx (transaction_a), open_blocks, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status != 0) - { - auto status (mdb_del (env.tx (transaction_a), change_blocks, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0); - } - } - } - } - } -} - -bool nano::mdb_store::block_exists (nano::transaction const & transaction_a, nano::block_type type, nano::block_hash const & hash_a) -{ - auto exists (false); - nano::mdb_val junk; - - switch (type) - { - case nano::block_type::send: - { - auto status (mdb_get (env.tx (transaction_a), send_blocks, nano::mdb_val (hash_a), junk)); - assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - break; - } - case nano::block_type::receive: - { - auto status (mdb_get (env.tx (transaction_a), receive_blocks, nano::mdb_val (hash_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - break; - } - case nano::block_type::open: - { - auto status (mdb_get (env.tx (transaction_a), open_blocks, nano::mdb_val (hash_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - break; - } - case nano::block_type::change: - { - auto status (mdb_get (env.tx (transaction_a), change_blocks, nano::mdb_val (hash_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - break; - } - case nano::block_type::state: - { - auto status (mdb_get (env.tx (transaction_a), state_blocks_v0, nano::mdb_val (hash_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - if (!exists) - { - auto status (mdb_get (env.tx (transaction_a), state_blocks_v1, nano::mdb_val (hash_a), junk)); - release_assert (status == 0 || status == MDB_NOTFOUND); - exists = status == 0; - } - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - break; - } - - return exists; -} - -nano::block_counts nano::mdb_store::block_count (nano::transaction const & transaction_a) -{ - nano::block_counts result; - result.send = count (transaction_a, send_blocks); - result.receive = count (transaction_a, receive_blocks); - result.open = count (transaction_a, open_blocks); - result.change = count (transaction_a, change_blocks); - result.state_v0 = count (transaction_a, state_blocks_v0); - result.state_v1 = count (transaction_a, state_blocks_v1); - return result; -} - -void nano::mdb_store::account_del (nano::transaction const & transaction_a, nano::account const & account_a) -{ - auto status1 (mdb_del (env.tx (transaction_a), accounts_v1, nano::mdb_val (account_a), nullptr)); - if (status1 != 0) - { - release_assert (status1 == MDB_NOTFOUND); - auto status2 (mdb_del (env.tx (transaction_a), accounts_v0, nano::mdb_val (account_a), nullptr)); - release_assert (status2 == 0); - } -} - -bool nano::mdb_store::account_get (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) -{ - nano::mdb_val value; - auto status1 (mdb_get (env.tx (transaction_a), accounts_v1, nano::mdb_val (account_a), value)); - release_assert (status1 == 0 || status1 == MDB_NOTFOUND); - bool result (false); - nano::epoch epoch; - if (status1 == 0) - { - epoch = nano::epoch::epoch_1; - } - else - { - auto status2 (mdb_get (env.tx (transaction_a), accounts_v0, nano::mdb_val (account_a), value)); - release_assert (status2 == 0 || status2 == MDB_NOTFOUND); - if (status2 == 0) - { - epoch = nano::epoch::epoch_0; - } - else - { - result = true; - } - } - if (!result) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - info_a.epoch = epoch; - result = info_a.deserialize (stream); - } - return result; -} - -void nano::mdb_store::frontier_put (nano::transaction const & transaction_a, nano::block_hash const & block_a, nano::account const & account_a) -{ - auto status (mdb_put (env.tx (transaction_a), frontiers, nano::mdb_val (block_a), nano::mdb_val (account_a), 0)); - release_assert (status == 0); -} - -nano::account nano::mdb_store::frontier_get (nano::transaction const & transaction_a, nano::block_hash const & block_a) const -{ - nano::mdb_val value; - auto status (mdb_get (env.tx (transaction_a), frontiers, nano::mdb_val (block_a), value)); - release_assert (status == 0 || status == MDB_NOTFOUND); - nano::account result (0); - if (status == 0) - { - result = nano::uint256_union (value); - } - return result; -} - -void nano::mdb_store::frontier_del (nano::transaction const & transaction_a, nano::block_hash const & block_a) -{ - auto status (mdb_del (env.tx (transaction_a), frontiers, nano::mdb_val (block_a), nullptr)); - release_assert (status == 0); -} - -size_t nano::mdb_store::account_count (nano::transaction const & transaction_a) -{ - return count (transaction_a, { accounts_v0, accounts_v1 }); -} - -MDB_dbi nano::mdb_store::get_account_db (nano::epoch epoch_a) const -{ - MDB_dbi db; - switch (epoch_a) - { - case nano::epoch::invalid: - case nano::epoch::unspecified: - assert (false); - case nano::epoch::epoch_0: - db = accounts_v0; - break; - case nano::epoch::epoch_1: - db = accounts_v1; - break; - } - return db; -} - -MDB_dbi nano::mdb_store::get_pending_db (nano::epoch epoch_a) const -{ - MDB_dbi db; - switch (epoch_a) - { - case nano::epoch::invalid: - case nano::epoch::unspecified: - assert (false); - case nano::epoch::epoch_0: - db = pending_v0; - break; - case nano::epoch::epoch_1: - db = pending_v1; - break; - } - return db; -} - -void nano::mdb_store::account_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) -{ - auto status (mdb_put (env.tx (transaction_a), get_account_db (info_a.epoch), nano::mdb_val (account_a), nano::mdb_val (info_a), 0)); - release_assert (status == 0); -} - -void nano::mdb_store::pending_put (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) -{ - auto status (mdb_put (env.tx (transaction_a), get_pending_db (pending_a.epoch), nano::mdb_val (key_a), nano::mdb_val (pending_a), 0)); - release_assert (status == 0); -} - -void nano::mdb_store::pending_del (nano::transaction const & transaction_a, nano::pending_key const & key_a) -{ - auto status1 (mdb_del (env.tx (transaction_a), pending_v1, mdb_val (key_a), nullptr)); - if (status1 != 0) - { - release_assert (status1 == MDB_NOTFOUND); - auto status2 (mdb_del (env.tx (transaction_a), pending_v0, mdb_val (key_a), nullptr)); - release_assert (status2 == 0); - } -} - -bool nano::mdb_store::pending_get (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) -{ - nano::mdb_val value; - auto status1 (mdb_get (env.tx (transaction_a), pending_v1, mdb_val (key_a), value)); - release_assert (status1 == 0 || status1 == MDB_NOTFOUND); - bool result (false); - nano::epoch epoch; - if (status1 == 0) - { - epoch = nano::epoch::epoch_1; - } - else - { - auto status2 (mdb_get (env.tx (transaction_a), pending_v0, mdb_val (key_a), value)); - release_assert (status2 == 0 || status2 == MDB_NOTFOUND); - if (status2 == 0) - { - epoch = nano::epoch::epoch_0; - } - else - { - result = true; - } - } - if (!result) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - pending_a.epoch = epoch; - result = pending_a.deserialize (stream); - } - return result; -} - -nano::store_iterator nano::mdb_store::pending_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v0, pending_v1, mdb_val (key_a))); - return result; -} - -nano::store_iterator nano::mdb_store::pending_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v0, pending_v1)); - return result; -} - -nano::store_iterator nano::mdb_store::pending_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v0_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v0, mdb_val (key_a))); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v0_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v0)); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v0_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v1_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v1, mdb_val (key_a))); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v1_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, pending_v1)); - return result; -} - -nano::store_iterator nano::mdb_store::pending_v1_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -bool nano::mdb_store::block_info_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_info & block_info_a) const -{ - assert (!full_sideband (transaction_a)); - nano::mdb_val value; - auto status (mdb_get (env.tx (transaction_a), blocks_info, nano::mdb_val (hash_a), value)); - release_assert (status == 0 || status == MDB_NOTFOUND); - bool result (true); - if (status != MDB_NOTFOUND) - { - result = false; - assert (value.size () == sizeof (block_info_a.account.bytes) + sizeof (block_info_a.balance.bytes)); - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - auto error1 (nano::try_read (stream, block_info_a.account)); - (void)error1; - assert (!error1); - auto error2 (nano::try_read (stream, block_info_a.balance)); - (void)error2; - assert (!error2); - } - return result; -} - -nano::uint128_t nano::mdb_store::representation_get (nano::transaction const & transaction_a, nano::account const & account_a) -{ - nano::mdb_val value; - auto status (mdb_get (env.tx (transaction_a), representation, nano::mdb_val (account_a), value)); - release_assert (status == 0 || status == MDB_NOTFOUND); - nano::uint128_t result = 0; - if (status == 0) - { - nano::uint128_union rep; - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - auto error (nano::try_read (stream, rep)); - (void)error; - assert (!error); - result = rep.number (); - } - return result; -} - -void nano::mdb_store::representation_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::uint128_t const & representation_a) -{ - nano::uint128_union rep (representation_a); - auto status (mdb_put (env.tx (transaction_a), representation, nano::mdb_val (account_a), nano::mdb_val (rep), 0)); - release_assert (status == 0); -} - -void nano::mdb_store::unchecked_clear (nano::transaction const & transaction_a) -{ - auto status (mdb_drop (env.tx (transaction_a), unchecked, 0)); - release_assert (status == 0); -} - -void nano::mdb_store::unchecked_put (nano::transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) -{ - auto status (mdb_put (env.tx (transaction_a), unchecked, nano::mdb_val (key_a), nano::mdb_val (info_a), 0)); - release_assert (status == 0); -} - -std::shared_ptr nano::mdb_store::vote_get (nano::transaction const & transaction_a, nano::account const & account_a) -{ - nano::mdb_val value; - auto status (mdb_get (env.tx (transaction_a), vote, nano::mdb_val (account_a), value)); - release_assert (status == 0 || status == MDB_NOTFOUND); - if (status == 0) - { - std::shared_ptr result (value); - assert (result != nullptr); - return result; - } - return nullptr; -} - -void nano::mdb_store::unchecked_del (nano::transaction const & transaction_a, nano::unchecked_key const & key_a) -{ - auto status (mdb_del (env.tx (transaction_a), unchecked, nano::mdb_val (key_a), nullptr)); - release_assert (status == 0 || status == MDB_NOTFOUND); -} - -size_t nano::mdb_store::unchecked_count (nano::transaction const & transaction_a) -{ - return count (transaction_a, unchecked); -} - -size_t nano::mdb_store::count (nano::transaction const & transaction_a, MDB_dbi db_a) const -{ - MDB_stat stats; - auto status (mdb_stat (env.tx (transaction_a), db_a, &stats)); - release_assert (status == 0); - return (stats.ms_entries); -} - -void nano::mdb_store::online_weight_put (nano::transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) -{ - auto status (mdb_put (env.tx (transaction_a), online_weight, nano::mdb_val (time_a), nano::mdb_val (amount_a), 0)); - release_assert (status == 0); -} - -void nano::mdb_store::online_weight_del (nano::transaction const & transaction_a, uint64_t time_a) -{ - auto status (mdb_del (env.tx (transaction_a), online_weight, nano::mdb_val (time_a), nullptr)); - release_assert (status == 0); -} - -nano::store_iterator nano::mdb_store::online_weight_begin (nano::transaction const & transaction_a) -{ - return nano::store_iterator (std::make_unique> (transaction_a, online_weight)); -} - -nano::store_iterator nano::mdb_store::online_weight_end () -{ - return nano::store_iterator (nullptr); -} - -size_t nano::mdb_store::online_weight_count (nano::transaction const & transaction_a) const -{ - return count (transaction_a, online_weight); -} - -void nano::mdb_store::online_weight_clear (nano::transaction const & transaction_a) -{ - auto status (mdb_drop (env.tx (transaction_a), online_weight, 0)); - release_assert (status == 0); -} - -void nano::mdb_store::flush (nano::transaction const & transaction_a) -{ - { - std::lock_guard lock (cache_mutex); - vote_cache_l1.swap (vote_cache_l2); - vote_cache_l1.clear (); - } - for (auto i (vote_cache_l2.begin ()), n (vote_cache_l2.end ()); i != n; ++i) - { - std::vector vector; - { - nano::vectorstream stream (vector); - i->second->serialize (stream); - } - auto status1 (mdb_put (env.tx (transaction_a), vote, nano::mdb_val (i->first), nano::mdb_val (vector.size (), vector.data ()), 0)); - release_assert (status1 == 0); - } -} - -size_t nano::mdb_store::count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const -{ - size_t total_count = 0; - for (auto db : dbs_a) - { - total_count += count (transaction_a, db); - } - return total_count; -} - -nano::store_iterator nano::mdb_store::latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v0, accounts_v1, nano::mdb_val (account_a))); - return result; -} - -nano::store_iterator nano::mdb_store::latest_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); - return result; -} - -nano::store_iterator nano::mdb_store::latest_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v0_begin (nano::transaction const & transaction_a, nano::account const & account_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v0, nano::mdb_val (account_a))); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v0_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v0)); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v0_end () -{ - nano::store_iterator result (nullptr); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v1_begin (nano::transaction const & transaction_a, nano::account const & account_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v1, nano::mdb_val (account_a))); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v1_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, accounts_v1)); - return result; -} - -nano::store_iterator nano::mdb_store::latest_v1_end () -{ - nano::store_iterator result (nullptr); - return result; -} diff --git a/nano/node/lmdb.hpp b/nano/node/lmdb.hpp deleted file mode 100644 index f62d31536e..0000000000 --- a/nano/node/lmdb.hpp +++ /dev/null @@ -1,375 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include - -namespace nano -{ -class mdb_env; - -class mdb_txn_callbacks -{ -public: - // clang-format off - std::function txn_start{ [] (const nano::transaction_impl *) {} }; - std::function txn_end{ [] (const nano::transaction_impl *) {} }; - // clang-format on -}; - -class read_mdb_txn final : public read_transaction_impl -{ -public: - read_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks); - ~read_mdb_txn (); - void reset () const override; - void renew () const override; - void * get_handle () const override; - MDB_txn * handle; - mdb_txn_callbacks txn_callbacks; -}; - -class write_mdb_txn final : public write_transaction_impl -{ -public: - write_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks); - ~write_mdb_txn (); - void commit () const override; - void renew () override; - void * get_handle () const override; - MDB_txn * handle; - nano::mdb_env const & env; - mdb_txn_callbacks txn_callbacks; -}; -/** - * RAII wrapper for MDB_env - */ -class mdb_env final -{ -public: - mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024); - ~mdb_env (); - operator MDB_env * () const; - // clang-format off - nano::read_transaction tx_begin_read (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const; - nano::write_transaction tx_begin_write (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const; - MDB_txn * tx (nano::transaction const & transaction_a) const; - // clang-format on - MDB_env * environment; -}; - -using mdb_val = db_val; - -template -class mdb_iterator : public store_iterator_impl -{ -public: - mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, nano::epoch = nano::epoch::unspecified); - mdb_iterator (std::nullptr_t, nano::epoch = nano::epoch::unspecified); - mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a, nano::epoch = nano::epoch::unspecified); - mdb_iterator (nano::mdb_iterator && other_a); - mdb_iterator (nano::mdb_iterator const &) = delete; - ~mdb_iterator (); - nano::store_iterator_impl & operator++ () override; - std::pair * operator-> (); - bool operator== (nano::store_iterator_impl const & other_a) const override; - bool is_end_sentinal () const override; - void fill (std::pair &) const override; - void clear (); - nano::mdb_iterator & operator= (nano::mdb_iterator && other_a); - nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; - MDB_cursor * cursor; - std::pair current; - -private: - MDB_txn * tx (nano::transaction const &) const; -}; - -/** - * Iterates the key/value pairs of two stores merged together - */ -template -class mdb_merge_iterator : public store_iterator_impl -{ -public: - mdb_merge_iterator (nano::transaction const &, MDB_dbi, MDB_dbi); - mdb_merge_iterator (std::nullptr_t); - mdb_merge_iterator (nano::transaction const &, MDB_dbi, MDB_dbi, MDB_val const &); - mdb_merge_iterator (nano::mdb_merge_iterator &&); - mdb_merge_iterator (nano::mdb_merge_iterator const &) = delete; - ~mdb_merge_iterator (); - nano::store_iterator_impl & operator++ () override; - std::pair * operator-> (); - bool operator== (nano::store_iterator_impl const &) const override; - bool is_end_sentinal () const override; - void fill (std::pair &) const override; - void clear (); - nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator &&) = default; - nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator const &) = delete; - -private: - nano::mdb_iterator & least_iterator () const; - std::unique_ptr> impl1; - std::unique_ptr> impl2; -}; - -class logging_mt; -/** - * mdb implementation of the block store - */ -class mdb_store : public block_store_partial -{ -public: - using block_store_partial::block_exists; - using block_store_partial::unchecked_put; - - mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512); - nano::write_transaction tx_begin_write () override; - nano::read_transaction tx_begin_read () override; - - std::shared_ptr block_random (nano::transaction const &) override; - void block_del (nano::transaction const &, nano::block_hash const &) override; - bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) override; - nano::block_counts block_count (nano::transaction const &) override; - - void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) override; - nano::account frontier_get (nano::transaction const &, nano::block_hash const &) const override; - void frontier_del (nano::transaction const &, nano::block_hash const &) override; - - void account_put (nano::transaction const &, nano::account const &, nano::account_info const &) override; - bool account_get (nano::transaction const &, nano::account const &, nano::account_info &) override; - void account_del (nano::transaction const &, nano::account const &) override; - size_t account_count (nano::transaction const &) override; - nano::store_iterator latest_v0_begin (nano::transaction const &, nano::account const &) override; - nano::store_iterator latest_v0_begin (nano::transaction const &) override; - nano::store_iterator latest_v0_end () override; - nano::store_iterator latest_v1_begin (nano::transaction const &, nano::account const &) override; - nano::store_iterator latest_v1_begin (nano::transaction const &) override; - nano::store_iterator latest_v1_end () override; - nano::store_iterator latest_begin (nano::transaction const &, nano::account const &) override; - nano::store_iterator latest_begin (nano::transaction const &) override; - nano::store_iterator latest_end () override; - - void pending_put (nano::transaction const &, nano::pending_key const &, nano::pending_info const &) override; - void pending_del (nano::transaction const &, nano::pending_key const &) override; - bool pending_get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) override; - nano::store_iterator pending_v0_begin (nano::transaction const &, nano::pending_key const &) override; - nano::store_iterator pending_v0_begin (nano::transaction const &) override; - nano::store_iterator pending_v0_end () override; - nano::store_iterator pending_v1_begin (nano::transaction const &, nano::pending_key const &) override; - nano::store_iterator pending_v1_begin (nano::transaction const &) override; - nano::store_iterator pending_v1_end () override; - nano::store_iterator pending_begin (nano::transaction const &, nano::pending_key const &) override; - nano::store_iterator pending_begin (nano::transaction const &) override; - nano::store_iterator pending_end () override; - - bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override; - nano::epoch block_version (nano::transaction const &, nano::block_hash const &) override; - - nano::uint128_t representation_get (nano::transaction const &, nano::account const &) override; - void representation_put (nano::transaction const &, nano::account const &, nano::uint128_t const &) override; - nano::store_iterator representation_begin (nano::transaction const &) override; - nano::store_iterator representation_end () override; - - void unchecked_clear (nano::transaction const &) override; - void unchecked_put (nano::transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) override; - void unchecked_del (nano::transaction const &, nano::unchecked_key const &) override; - nano::store_iterator unchecked_begin (nano::transaction const &) override; - nano::store_iterator unchecked_begin (nano::transaction const &, nano::unchecked_key const &) override; - nano::store_iterator unchecked_end () override; - size_t unchecked_count (nano::transaction const &) override; - - // Return latest vote for an account from store - std::shared_ptr vote_get (nano::transaction const &, nano::account const &) override; - void flush (nano::transaction const &) override; - nano::store_iterator> vote_begin (nano::transaction const &) override; - nano::store_iterator> vote_end () override; - - void online_weight_put (nano::transaction const &, uint64_t, nano::amount const &) override; - void online_weight_del (nano::transaction const &, uint64_t) override; - nano::store_iterator online_weight_begin (nano::transaction const &) override; - nano::store_iterator online_weight_end () override; - size_t online_weight_count (nano::transaction const &) const override; - void online_weight_clear (nano::transaction const &) override; - - void version_put (nano::transaction const &, int) override; - int version_get (nano::transaction const &) const override; - - void peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - bool peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const override; - void peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - size_t peer_count (nano::transaction const & transaction_a) const override; - void peer_clear (nano::transaction const & transaction_a) override; - - nano::store_iterator peers_begin (nano::transaction const & transaction_a) override; - nano::store_iterator peers_end () override; - - MDB_dbi get_account_db (nano::epoch epoch_a) const; - void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override; - - nano::logger_mt & logger; - - nano::mdb_env env; - - /** - * Maps head block to owning account - * nano::block_hash -> nano::account - */ - MDB_dbi frontiers{ 0 }; - - /** - * Maps account v1 to account information, head, rep, open, balance, timestamp and block count. - * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t - */ - MDB_dbi accounts_v0{ 0 }; - - /** - * Maps account v0 to account information, head, rep, open, balance, timestamp and block count. - * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t - */ - MDB_dbi accounts_v1{ 0 }; - - /** - * Maps block hash to send block. - * nano::block_hash -> nano::send_block - */ - MDB_dbi send_blocks{ 0 }; - - /** - * Maps block hash to receive block. - * nano::block_hash -> nano::receive_block - */ - MDB_dbi receive_blocks{ 0 }; - - /** - * Maps block hash to open block. - * nano::block_hash -> nano::open_block - */ - MDB_dbi open_blocks{ 0 }; - - /** - * Maps block hash to change block. - * nano::block_hash -> nano::change_block - */ - MDB_dbi change_blocks{ 0 }; - - /** - * Maps block hash to v0 state block. - * nano::block_hash -> nano::state_block - */ - MDB_dbi state_blocks_v0{ 0 }; - - /** - * Maps block hash to v1 state block. - * nano::block_hash -> nano::state_block - */ - MDB_dbi state_blocks_v1{ 0 }; - - /** - * Maps min_version 0 (destination account, pending block) to (source account, amount). - * nano::account, nano::block_hash -> nano::account, nano::amount - */ - MDB_dbi pending_v0{ 0 }; - - /** - * Maps min_version 1 (destination account, pending block) to (source account, amount). - * nano::account, nano::block_hash -> nano::account, nano::amount - */ - MDB_dbi pending_v1{ 0 }; - - /** - * Maps block hash to account and balance. - * block_hash -> nano::account, nano::amount - */ - MDB_dbi blocks_info{ 0 }; - - /** - * Representative weights. - * nano::account -> nano::uint128_t - */ - MDB_dbi representation{ 0 }; - - /** - * Unchecked bootstrap blocks info. - * nano::block_hash -> nano::unchecked_info - */ - MDB_dbi unchecked{ 0 }; - - /** - * Highest vote observed for account. - * nano::account -> uint64_t - */ - MDB_dbi vote{ 0 }; - - /** - * Samples of online vote weight - * uint64_t -> nano::amount - */ - MDB_dbi online_weight{ 0 }; - - /** - * Meta information about block store, such as versions. - * nano::uint256_union (arbitrary key) -> blob - */ - MDB_dbi meta{ 0 }; - - /* - * Endpoints for peers - * nano::endpoint_key -> no_value - */ - MDB_dbi peers{ 0 }; - -private: - MDB_dbi block_database (nano::block_type, nano::epoch); - template - std::shared_ptr block_random (nano::transaction const &, MDB_dbi); - boost::optional block_raw_get_by_type (nano::transaction const &, nano::block_hash const &, nano::block_type &) const override; - void block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) override; - void clear (MDB_dbi); - bool do_upgrades (nano::write_transaction &, size_t); - void upgrade_v1_to_v2 (nano::transaction const &); - void upgrade_v2_to_v3 (nano::transaction const &); - void upgrade_v3_to_v4 (nano::transaction const &); - void upgrade_v4_to_v5 (nano::transaction const &); - void upgrade_v5_to_v6 (nano::transaction const &); - void upgrade_v6_to_v7 (nano::transaction const &); - void upgrade_v7_to_v8 (nano::transaction const &); - void upgrade_v8_to_v9 (nano::transaction const &); - void upgrade_v9_to_v10 (nano::transaction const &); - void upgrade_v10_to_v11 (nano::transaction const &); - void upgrade_v11_to_v12 (nano::transaction const &); - void upgrade_v12_to_v13 (nano::write_transaction &, size_t); - void upgrade_v13_to_v14 (nano::transaction const &); - MDB_dbi get_pending_db (nano::epoch epoch_a) const; - void open_databases (bool &, nano::transaction const &, unsigned); - nano::mdb_txn_tracker mdb_txn_tracker; - nano::mdb_txn_callbacks create_txn_callbacks (); - bool txn_tracking_enabled; - static int constexpr version{ 14 }; - - size_t count (nano::transaction const &, MDB_dbi) const; - size_t count (nano::transaction const &, std::initializer_list) const; -}; -class wallet_value -{ -public: - wallet_value () = default; - wallet_value (nano::mdb_val const &); - wallet_value (nano::uint256_union const &, uint64_t); - nano::mdb_val val () const; - nano::private_key key; - uint64_t work; -}; -} diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 9166c6abc0..528423aaf8 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -135,6 +135,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "confirmation_height", flags, &confirmation_height) != 0; if (!full_sideband (transaction_a)) { error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks_info", flags, &blocks_info) != 0; @@ -173,6 +174,8 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size case 13: upgrade_v13_to_v14 (transaction_a); case 14: + upgrade_v14_to_v15 (transaction_a); + case 15: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -443,7 +446,7 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_ nano::store_iterator i (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); nano::store_iterator n (nullptr); - std::vector> account_infos; + std::vector> account_infos; account_infos.reserve (account_count (transaction_a)); for (; i != n; ++i) { @@ -453,12 +456,13 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_ { confirmation_height = 1; } - account_infos.emplace_back (i->first, nano::account_info{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, account_info_v13.epoch }); + account_infos.emplace_back (i->first, nano::account_info_v14{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, account_info_v13.epoch }); } for (auto const & account_info : account_infos) { - account_put (transaction_a, account_info.first, account_info.second); + auto status1 (mdb_put (env.tx (transaction_a), table_to_dbi (get_account_db (account_info.second.epoch)), nano::mdb_val (account_info.first), nano::mdb_val (account_info.second), 0)); + release_assert (status1 == 0); } logger.always_log ("Completed confirmation height upgrade"); @@ -468,6 +472,30 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_ release_assert (!error || error == MDB_NOTFOUND); } +void nano::mdb_store::upgrade_v14_to_v15 (nano::transaction const & transaction_a) +{ + version_put (transaction_a, 15); + + // Move confirmation height from account_info database to its own table + std::vector> account_infos; + account_infos.reserve (account_count (transaction_a)); + std::vector> confirmation_heights; + + nano::store_iterator i (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); + nano::store_iterator n (nullptr); + for (; i != n; ++i) + { + auto const & account_info_v14 (i->second); + account_infos.emplace_back (i->first, nano::account_info{ account_info_v14.head, account_info_v14.rep_block, account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, account_info_v14.epoch }); + confirmation_height_put (transaction_a, i->first, i->second.confirmation_height); + } + + for (auto const & account_info : account_infos) + { + account_put (transaction_a, account_info.first, account_info.second); + } +} + void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a) { nano::uint256_union version_key (1); @@ -595,6 +623,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const return meta; case tables::peers: return peers; + case tables::confirmation_height: + return confirmation_height; default: release_assert (false); return peers; diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 0c8f66d6a5..109c85b50e 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -154,6 +154,12 @@ class mdb_store : public block_store_partial */ MDB_dbi peers{ 0 }; + /* + * Confirmation height of an account + * nano::account -> uint64_t + */ + MDB_dbi confirmation_height{ 0 }; + bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const; @@ -198,6 +204,7 @@ class mdb_store : public block_store_partial void upgrade_v11_to_v12 (nano::transaction const &); void upgrade_v12_to_v13 (nano::write_transaction &, size_t); void upgrade_v13_to_v14 (nano::transaction const &); + void upgrade_v14_to_v15 (nano::transaction const &); void open_databases (bool &, nano::transaction const &, unsigned); int drop (nano::transaction const & transaction_a, tables table_a) override; diff --git a/nano/node/lmdb_txn_tracker.cpp b/nano/node/lmdb_txn_tracker.cpp deleted file mode 100644 index 7b0cf1d894..0000000000 --- a/nano/node/lmdb_txn_tracker.cpp +++ /dev/null @@ -1,161 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -// Some builds (mac) fail due to "Boost.Stacktrace requires `_Unwind_Backtrace` function". -#ifndef _WIN32 -#ifndef _GNU_SOURCE -#define BEFORE_GNU_SOURCE 0 -#define _GNU_SOURCE -#else -#define BEFORE_GNU_SOURCE 1 -#endif -#endif -// On Windows this include defines min/max macros, so keep below other includes -// to reduce conflicts with other std functions -#include -#ifndef _WIN32 -#if !BEFORE_GNU_SOURCE -#undef _GNU_SOURCE -#endif -#endif - -namespace -{ -class matches_txn -{ -public: - matches_txn (const nano::transaction_impl * transaction_impl_a) : - transaction_impl (transaction_impl_a) - { - } - - bool operator() (nano::mdb_txn_stats const & mdb_txn_stats) - { - return (mdb_txn_stats.transaction_impl == transaction_impl); - } - -private: - const nano::transaction_impl * transaction_impl; -}; -} - -nano::mdb_txn_tracker::mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a) : -logger (logger_a), -txn_tracking_config (txn_tracking_config_a), -block_processor_batch_max_time (block_processor_batch_max_time_a) -{ -} - -void nano::mdb_txn_tracker::serialize_json (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time) -{ - // Copying is cheap compared to generating the stack trace strings, so reduce time holding the mutex - std::vector copy_stats; - { - std::lock_guard guard (mutex); - copy_stats = stats; - } - - // Get the time difference now as creating stacktraces (Debug/Windows for instance) can take a while so results won't be as accurate - std::vector times_since_start; - times_since_start.reserve (copy_stats.size ()); - // clang-format off - std::transform (copy_stats.cbegin (), copy_stats.cend (), std::back_inserter (times_since_start), [] (const auto & stat) { - return stat.timer.since_start (); - }); - // clang-format on - assert (times_since_start.size () == copy_stats.size ()); - - for (size_t i = 0; i < times_since_start.size (); ++i) - { - auto const & stat = copy_stats[i]; - auto time_held_open = times_since_start[i]; - - if ((stat.is_write () && time_held_open >= min_write_time) || (!stat.is_write () && time_held_open >= min_read_time)) - { - nano::jsonconfig mdb_lock_config; - - mdb_lock_config.put ("thread", stat.thread_name); - mdb_lock_config.put ("time_held_open", time_held_open.count ()); - mdb_lock_config.put ("write", stat.is_write ()); - - boost::property_tree::ptree stacktrace_config; - for (auto frame : *stat.stacktrace) - { - nano::jsonconfig frame_json; - frame_json.put ("name", frame.name ()); - frame_json.put ("address", frame.address ()); - frame_json.put ("source_file", frame.source_file ()); - frame_json.put ("source_line", frame.source_line ()); - stacktrace_config.push_back (std::make_pair ("", frame_json.get_tree ())); - } - - nano::jsonconfig stack (stacktrace_config); - mdb_lock_config.put_child ("stacktrace", stack); - json.push_back (std::make_pair ("", mdb_lock_config.get_tree ())); - } - } -} - -void nano::mdb_txn_tracker::output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const -{ - // Only output them if transactions were held for longer than a certain period of time - auto is_write = mdb_txn_stats.is_write (); - auto time_open = mdb_txn_stats.timer.since_start (); - - auto should_ignore = false; - // Reduce noise in log files by removing any entries from the block processor (if enabled) which are less than the max batch time (+ a few second buffer) because these are expected writes during bootstrapping. - auto is_below_max_time = time_open <= (block_processor_batch_max_time + std::chrono::seconds (3)); - bool is_blk_processing_thread = mdb_txn_stats.thread_name == nano::thread_role::get_string (nano::thread_role::name::block_processing); - if (txn_tracking_config.ignore_writes_below_block_processor_max_time && is_blk_processing_thread && is_write && is_below_max_time) - { - should_ignore = true; - } - - if (!should_ignore && ((is_write && time_open >= txn_tracking_config.min_write_txn_time) || (!is_write && time_open >= txn_tracking_config.min_read_txn_time))) - { - assert (mdb_txn_stats.stacktrace); - logger.always_log (boost::str (boost::format ("%1%ms %2% held on thread %3%\n%4%") % mdb_txn_stats.timer.since_start ().count () % (is_write ? "write lock" : "read") % mdb_txn_stats.thread_name % *mdb_txn_stats.stacktrace)); - } -} - -void nano::mdb_txn_tracker::add (const nano::transaction_impl * transaction_impl) -{ - std::lock_guard guard (mutex); - // clang-format off - assert (std::find_if (stats.cbegin (), stats.cend (), matches_txn (transaction_impl)) == stats.cend ()); - // clang-format on - stats.emplace_back (transaction_impl); -} - -/** Can be called without error if transaction does not exist */ -void nano::mdb_txn_tracker::erase (const nano::transaction_impl * transaction_impl) -{ - std::lock_guard guard (mutex); - // clang-format off - auto it = std::find_if (stats.begin (), stats.end (), matches_txn (transaction_impl)); - // clang-format on - if (it != stats.end ()) - { - output_finished (*it); - it->timer.stop (); - stats.erase (it); - } -} - -nano::mdb_txn_stats::mdb_txn_stats (const nano::transaction_impl * transaction_impl) : -transaction_impl (transaction_impl), -thread_name (nano::thread_role::get_string ()), -stacktrace (std::make_shared ()) -{ - timer.start (); -} - -bool nano::mdb_txn_stats::is_write () const -{ - return (dynamic_cast (transaction_impl) != nullptr); -} diff --git a/nano/node/lmdb_txn_tracker.hpp b/nano/node/lmdb_txn_tracker.hpp deleted file mode 100644 index 1eb3e3dd16..0000000000 --- a/nano/node/lmdb_txn_tracker.hpp +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - -namespace nano -{ -class transaction_impl; -class logger_mt; - -class mdb_txn_stats -{ -public: - mdb_txn_stats (const nano::transaction_impl * transaction_impl_a); - bool is_write () const; - nano::timer timer; - const nano::transaction_impl * transaction_impl; - std::string thread_name; - - // Smart pointer so that we don't need the full definition which causes min/max issues on Windows - std::shared_ptr stacktrace; -}; - -class mdb_txn_tracker -{ -public: - mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a); - void serialize_json (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time); - void add (const nano::transaction_impl * transaction_impl); - void erase (const nano::transaction_impl * transaction_impl); - -private: - std::mutex mutex; - std::vector stats; - nano::logger_mt & logger; - nano::txn_tracking_config txn_tracking_config; - std::chrono::milliseconds block_processor_batch_max_time; - - void output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const; -}; -} diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 9157c32e85..7a2b3351f4 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -107,10 +107,9 @@ void enable_ipc_transport_tcp (nano::ipc::ipc_config_tcp_socket & transport_tcp) void reset_confirmation_height (nano::block_store & store, nano::account const & account) { auto transaction = store.tx_begin_write (); - nano::account_info account_info; - store.account_get (transaction, account, account_info); - account_info.confirmation_height = 0; - store.account_put (transaction, account, account_info); + uint64_t confirmation_height; + store.confirmation_height_get (transaction, account, confirmation_height); + store.confirmation_height_clear (transaction, account, confirmation_height); } void check_block_response_count (nano::system & system, nano::rpc & rpc, boost::property_tree::ptree & request, uint64_t size_count) @@ -1212,7 +1211,8 @@ TEST (rpc, frontier) { nano::keypair key; source[key.pub] = key.prv.data; - system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); + system.nodes[0]->store.confirmation_height_put (transaction, key.pub, 0); + system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } nano::keypair key; @@ -1259,7 +1259,8 @@ TEST (rpc, frontier_limited) { nano::keypair key; source[key.pub] = key.prv.data; - system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); + system.nodes[0]->store.confirmation_height_put (transaction, key.pub, 0); + system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } nano::keypair key; @@ -1296,7 +1297,8 @@ TEST (rpc, frontier_startpoint) { nano::keypair key; source[key.pub] = key.prv.data; - system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); + system.nodes[0]->store.confirmation_height_put (transaction, key.pub, 0); + system.nodes[0]->store.account_put (transaction, key.pub, nano::account_info (key.prv.data, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } nano::keypair key; @@ -4361,11 +4363,8 @@ TEST (rpc, account_info) auto time (nano::seconds_since_epoch ()); { - auto transaction = system.nodes[0]->store.tx_begin_write (); - nano::account_info account_info; - ASSERT_FALSE (node1.store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - account_info.confirmation_height = 1; - node1.store.account_put (transaction, nano::test_genesis_key.pub, account_info); + auto transaction = node1.store.tx_begin_write (); + node1.store.confirmation_height_put (transaction, nano::test_genesis_key.pub, 1); } enable_ipc_transport_tcp (node1.config.ipc_config.transport_tcp); diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 023c7a8b5e..a80300a71d 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -52,6 +52,11 @@ class db_val { } + db_val (nano::account_info_v14 const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + } + db_val (nano::pending_info const & val_a) : db_val (sizeof (val_a.source) + sizeof (val_a.amount), const_cast (&val_a)) { @@ -125,6 +130,15 @@ class db_val return result; } + explicit operator nano::account_info_v14 () const + { + nano::account_info_v14 result; + result.epoch = epoch; + assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + explicit operator nano::block_info () const { nano::block_info result; @@ -548,7 +562,8 @@ class block_store vote, online_weight, meta, - peers + peers, + confirmation_height }; virtual ~block_store () = default; @@ -575,7 +590,7 @@ class block_store virtual void account_del (nano::transaction const &, nano::account const &) = 0; virtual bool account_exists (nano::transaction const &, nano::account const &) = 0; virtual size_t account_count (nano::transaction const &) = 0; - virtual void confirmation_height_clear (nano::transaction const &, nano::account const & account, nano::account_info const & account_info) = 0; + virtual void confirmation_height_clear (nano::transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0; virtual void confirmation_height_clear (nano::transaction const &) = 0; virtual uint64_t cemented_count (nano::transaction const &) = 0; virtual nano::store_iterator latest_v0_begin (nano::transaction const &, nano::account const &) = 0; @@ -653,6 +668,16 @@ class block_store virtual void peer_clear (nano::transaction const & transaction_a) = 0; virtual nano::store_iterator peers_begin (nano::transaction const & transaction_a) = 0; virtual nano::store_iterator peers_end () = 0; + + virtual void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0; + virtual bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) = 0; + virtual bool confirmation_height_exists (nano::transaction const & transaction_a, nano::account const & account_a) const = 0; + virtual void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) = 0; + virtual uint64_t confirmation_height_count (nano::transaction const & transaction_a) = 0; + virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) = 0; + virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) = 0; + virtual nano::store_iterator confirmation_height_end () = 0; + virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; virtual std::mutex & get_cache_mutex () = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index 7ac5e4d4c5..1c58ad1c04 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -30,7 +30,8 @@ class block_store_partial : public block_store assert (latest_v1_begin (transaction_a) == latest_v1_end ()); nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch ()); block_put (transaction_a, hash_l, *genesis_a.open, sideband); - account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, 1, nano::epoch::epoch_0 }); + confirmation_height_put (transaction_a, network_params.ledger.genesis_account, 1); + account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); representation_put (transaction_a, network_params.ledger.genesis_account, std::numeric_limits::max ()); frontier_put (transaction_a, hash_l, network_params.ledger.genesis_account); } @@ -58,19 +59,17 @@ class block_store_partial : public block_store return iterator != latest_end () && nano::account (iterator->first) == account_a; } - void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, nano::account_info const & account_info) override + void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override { - nano::account_info info_copy (account_info); - if (info_copy.confirmation_height > 0) + if (existing_confirmation_height > 0) { - info_copy.confirmation_height = 0; - account_put (transaction_a, account, info_copy); + confirmation_height_put (transaction_a, account, 0); } } void confirmation_height_clear (nano::transaction const & transaction_a) override { - for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) + for (auto i (confirmation_height_begin (transaction_a)), n (confirmation_height_end ()); i != n; ++i) { confirmation_height_clear (transaction_a, i->first, i->second); } @@ -256,10 +255,9 @@ class block_store_partial : public block_store uint64_t cemented_count (nano::transaction const & transaction_a) override { uint64_t sum = 0; - for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) + for (auto i (confirmation_height_begin (transaction_a)), n (confirmation_height_end ()); i != n; ++i) { - nano::account_info const & info (i->second); - sum += info.confirmation_height; + sum += i->second; } return sum; } @@ -346,7 +344,7 @@ class block_store_partial : public block_store nano::store_iterator peers_end () override { - return nano::store_iterator (nano::store_iterator (nullptr)); + return nano::store_iterator (nullptr); } nano::store_iterator pending_end () override @@ -384,6 +382,11 @@ class block_store_partial : public block_store return nano::store_iterator (nullptr); } + nano::store_iterator confirmation_height_end () override + { + return nano::store_iterator (nullptr); + } + std::mutex & get_cache_mutex () override { return cache_mutex; @@ -634,6 +637,8 @@ class block_store_partial : public block_store void account_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override { + // Check we are still in sync with other tables + assert (confirmation_height_exists (transaction_a, account_a)); nano::db_val info (info_a); auto status = put (transaction_a, get_account_db (info_a.epoch), account_a, info); release_assert (success (status)); @@ -730,7 +735,7 @@ class block_store_partial : public block_store release_assert (success (status)); } - int exists (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key_a) const + bool exists (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key_a) const { return static_cast (*this).exists (transaction_a, table_a, key_a); } @@ -803,6 +808,42 @@ class block_store_partial : public block_store return result; } + uint64_t confirmation_height_count (nano::transaction const & transaction_a) override + { + return count (transaction_a, tables::confirmation_height); + } + + void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override + { + nano::db_val confirmation_height (confirmation_height_a); + auto status = put (transaction_a, tables::confirmation_height, account_a, confirmation_height); + release_assert (success (status)); + } + + bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) override + { + nano::db_val value; + auto status = get (transaction_a, tables::confirmation_height, nano::db_val (account_a), value); + release_assert (success (status) || !not_found (status)); + confirmation_height_a = 0; + if (success (status)) + { + confirmation_height_a = static_cast (value); + } + return (!success (status)); + } + + void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) override + { + auto status (del (transaction_a, tables::confirmation_height, nano::db_val (account_a))); + release_assert (success (status)); + } + + bool confirmation_height_exists (nano::transaction const & transaction_a, nano::account const & account_a) const override + { + return exists (transaction_a, tables::confirmation_height, nano::db_val (account_a)); + } + nano::store_iterator latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) override { return make_merge_iterator (transaction_a, tables::accounts_v0, tables::accounts_v1, nano::db_val (account_a)); @@ -893,6 +934,16 @@ class block_store_partial : public block_store return make_iterator (transaction_a, tables::peers); } + nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) override + { + return make_iterator (transaction_a, tables::confirmation_height, nano::db_val (account_a)); + } + + nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) override + { + return make_iterator (transaction_a, tables::confirmation_height); + } + size_t unchecked_count (nano::transaction const & transaction_a) override { return count (transaction_a, tables::unchecked); diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index d6b07c94cb..96e063063e 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -174,14 +174,13 @@ void nano::serialize_block (nano::stream & stream_a, nano::block const & block_a block_a.serialize (stream_a); } -nano::account_info::account_info (nano::block_hash const & head_a, nano::block_hash const & rep_block_a, nano::block_hash const & open_block_a, nano::amount const & balance_a, uint64_t modified_a, uint64_t block_count_a, uint64_t confirmation_height_a, nano::epoch epoch_a) : +nano::account_info::account_info (nano::block_hash const & head_a, nano::block_hash const & rep_block_a, nano::block_hash const & open_block_a, nano::amount const & balance_a, uint64_t modified_a, uint64_t block_count_a, nano::epoch epoch_a) : head (head_a), rep_block (rep_block_a), open_block (open_block_a), balance (balance_a), modified (modified_a), block_count (block_count_a), -confirmation_height (confirmation_height_a), epoch (epoch_a) { } @@ -197,7 +196,6 @@ bool nano::account_info::deserialize (nano::stream & stream_a) nano::read (stream_a, balance.bytes); nano::read (stream_a, modified); nano::read (stream_a, block_count); - nano::read (stream_a, confirmation_height); } catch (std::runtime_error const &) { @@ -209,7 +207,7 @@ bool nano::account_info::deserialize (nano::stream & stream_a) bool nano::account_info::operator== (nano::account_info const & other_a) const { - return head == other_a.head && rep_block == other_a.rep_block && open_block == other_a.open_block && balance == other_a.balance && modified == other_a.modified && block_count == other_a.block_count && confirmation_height == other_a.confirmation_height && epoch == other_a.epoch; + return head == other_a.head && rep_block == other_a.rep_block && open_block == other_a.open_block && balance == other_a.balance && modified == other_a.modified && block_count == other_a.block_count && epoch == other_a.epoch; } bool nano::account_info::operator!= (nano::account_info const & other_a) const @@ -225,8 +223,7 @@ size_t nano::account_info::db_size () const assert (reinterpret_cast (&open_block) + sizeof (open_block) == reinterpret_cast (&balance)); assert (reinterpret_cast (&balance) + sizeof (balance) == reinterpret_cast (&modified)); assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); - assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&confirmation_height)); - return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (confirmation_height); + return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count); } size_t nano::block_counts::sum () const diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 5f3e998c8e..4056cac214 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -84,7 +84,7 @@ class account_info final { public: account_info () = default; - account_info (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, uint64_t, epoch); + account_info (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, epoch); bool deserialize (nano::stream &); bool operator== (nano::account_info const &) const; bool operator!= (nano::account_info const &) const; @@ -96,7 +96,6 @@ class account_info final /** Seconds since posix epoch */ uint64_t modified{ 0 }; uint64_t block_count{ 0 }; - uint64_t confirmation_height{ 0 }; nano::epoch epoch{ nano::epoch::epoch_0 }; }; diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 9a09de9eb5..79274d438d 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -850,9 +850,12 @@ bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::bloc while (!error && store.block_exists (transaction_a, block_a)) { auto latest_error (store.account_get (transaction_a, account_l, account_info)); - (void)latest_error; assert (!latest_error); - if (block_account_height > account_info.confirmation_height) + uint64_t confirmation_height; + latest_error = store.confirmation_height_get (transaction_a, account_l, confirmation_height); + assert (!latest_error); + (void)latest_error; + if (block_account_height > confirmation_height) { auto block (store.block_get (transaction_a, account_info.head)); list_a.push_back (block); @@ -1008,10 +1011,20 @@ void nano::ledger::change_latest (nano::transaction const & transaction_a, nano: store.account_del (transaction_a, account_a); } info.epoch = epoch_a; + if (!store.confirmation_height_exists (transaction_a, account_a)) + { + // Make sure the accounts table is in sync with the confirmation height table + store.confirmation_height_put (transaction_a, account_a, 0); + } store.account_put (transaction_a, account_a, info); } else { + if (store.confirmation_height_exists (transaction_a, account_a)) + { + // Make sure the accounts table is in sync with the confirmation height table + store.confirmation_height_del (transaction_a, account_a); + } store.account_del (transaction_a, account_a); } } @@ -1062,12 +1075,11 @@ bool nano::ledger::block_confirmed (nano::transaction const & transaction_a, nan { auto confirmed (false); auto block_height (store.block_account_height (transaction_a, hash_a)); - if (block_height) // 0 indicates that the block doesn't exist + if (block_height > 0) // 0 indicates that the block doesn't exist { - auto account_l (account (transaction_a, hash_a)); - nano::account_info account_info; - release_assert (!store.account_get (transaction_a, account_l, account_info)); - confirmed = (account_info.confirmation_height >= block_height); + uint64_t confirmation_height; + release_assert (!store.confirmation_height_get (transaction_a, account (transaction_a, hash_a), confirmation_height)); + confirmed = (confirmation_height >= block_height); } return confirmed; } diff --git a/nano/secure/versioning.cpp b/nano/secure/versioning.cpp index dca7dde989..6ab77518f7 100644 --- a/nano/secure/versioning.cpp +++ b/nano/secure/versioning.cpp @@ -68,3 +68,27 @@ size_t nano::account_info_v13::db_size () const assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count); } + +nano::account_info_v14::account_info_v14 (nano::block_hash const & head_a, nano::block_hash const & rep_block_a, nano::block_hash const & open_block_a, nano::amount const & balance_a, uint64_t modified_a, uint64_t block_count_a, uint64_t confirmation_height_a, nano::epoch epoch_a) : +head (head_a), +rep_block (rep_block_a), +open_block (open_block_a), +balance (balance_a), +modified (modified_a), +block_count (block_count_a), +confirmation_height (confirmation_height_a), +epoch (epoch_a) +{ +} + +size_t nano::account_info_v14::db_size () const +{ + assert (reinterpret_cast (this) == reinterpret_cast (&head)); + assert (reinterpret_cast (&head) + sizeof (head) == reinterpret_cast (&rep_block)); + assert (reinterpret_cast (&rep_block) + sizeof (rep_block) == reinterpret_cast (&open_block)); + assert (reinterpret_cast (&open_block) + sizeof (open_block) == reinterpret_cast (&balance)); + assert (reinterpret_cast (&balance) + sizeof (balance) == reinterpret_cast (&modified)); + assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); + assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&confirmation_height)); + return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (confirmation_height); +} diff --git a/nano/secure/versioning.hpp b/nano/secure/versioning.hpp index 6616b27943..68221b5dba 100644 --- a/nano/secure/versioning.hpp +++ b/nano/secure/versioning.hpp @@ -45,7 +45,7 @@ class account_info_v13 final { public: account_info_v13 () = default; - account_info_v13 (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t block_count, nano::epoch epoch_a); + account_info_v13 (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::epoch); size_t db_size () const; nano::block_hash head{ 0 }; nano::block_hash rep_block{ 0 }; @@ -55,4 +55,19 @@ class account_info_v13 final uint64_t block_count{ 0 }; nano::epoch epoch{ nano::epoch::epoch_0 }; }; +class account_info_v14 final +{ +public: + account_info_v14 () = default; + account_info_v14 (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, uint64_t, nano::epoch); + size_t db_size () const; + nano::block_hash head{ 0 }; + nano::block_hash rep_block{ 0 }; + nano::block_hash open_block{ 0 }; + nano::amount balance{ 0 }; + uint64_t modified{ 0 }; + uint64_t block_count{ 0 }; + uint64_t confirmation_height{ 0 }; + nano::epoch epoch{ nano::epoch::epoch_0 }; +}; } diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index b22f937d8d..c520404596 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -176,6 +176,7 @@ TEST (store, load) { nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); + system.nodes[0]->store.confirmation_height_put (transaction, hash, 0); system.nodes[0]->store.account_put (transaction, hash, nano::account_info ()); } } @@ -514,16 +515,11 @@ TEST (confirmation_height, many_accounts_single_confirmation) { auto & account = i->first; auto & account_info = i->second; - if (account != last_keypair.pub) - { - ASSERT_EQ (2, account_info.confirmation_height); - ASSERT_EQ (2, account_info.block_count); - } - else - { - ASSERT_EQ (1, account_info.confirmation_height); - ASSERT_EQ (1, account_info.block_count); - } + auto count = (account != last_keypair.pub) ? 2 : 1; + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, account, confirmation_height)); + ASSERT_EQ (count, confirmation_height); + ASSERT_EQ (count, account_info.block_count); } ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_accounts * 2 - 2); @@ -642,11 +638,14 @@ TEST (confirmation_height, long_chains) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_EQ (num_blocks + 2, account_info.confirmation_height); + uint64_t confirmation_height; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); + ASSERT_EQ (num_blocks + 2, confirmation_height); ASSERT_EQ (num_blocks + 3, account_info.block_count); // Includes the unpocketed send ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_EQ (num_blocks + 1, account_info.confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); + ASSERT_EQ (num_blocks + 1, confirmation_height); ASSERT_EQ (num_blocks + 1, account_info.block_count); ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_blocks * 2 + 2);