Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Vacuum after upgrade #2370

Merged
merged 2 commits into from
Oct 30, 2019
Merged

Vacuum after upgrade #2370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e0fd26
Vacuum after upgrade
wezrule Oct 29, 2019
d78b273
Modify vacuum failed log message
wezrule Oct 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 57 additions & 12 deletions nano/node/lmdb/lmdb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,20 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
{
create_backup_file (env, path_a, logger_a);
}
auto transaction (tx_begin_write ());
open_databases (error, transaction, MDB_CREATE);
if (!error)
auto needs_vacuuming = false;
{
error |= do_upgrades (transaction, batch_size);
auto transaction (tx_begin_write ());
open_databases (error, transaction, MDB_CREATE);
if (!error)
{
error |= do_upgrades (transaction, needs_vacuuming, batch_size);
}
}

if (needs_vacuuming)
{
auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_max_dbs);
logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node");
}
}
else
Expand All @@ -81,6 +90,37 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
}
}

bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs)
{
// Vacuum the database. This is not a required step and may actually fail if there isn't enough storage space.
auto vacuum_path = path_a.parent_path () / "vacuumed.ldb";

auto vacuum_success = copy_db (vacuum_path);
if (vacuum_success)
{
// Need to close the database to release the file handle
mdb_env_close (env.environment);
env.environment = nullptr;

// Replace the ledger file with the vacuumed one
boost::filesystem::rename (vacuum_path, path_a);

// Set up the environment again
env.init (error, path_a, lmdb_max_dbs, true);
if (!error)
{
auto transaction (tx_begin_read ());
open_databases (error, transaction, 0);
}
}
else
{
// The vacuum file can be in an inconsistent state if there wasn't enough space to create it
boost::filesystem::remove (vacuum_path);
}
return vacuum_success;
}

void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time)
{
mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time);
Expand Down Expand Up @@ -150,7 +190,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const &
}
}

bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size_t batch_size)
bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool & needs_vacuuming, size_t batch_size_a)
{
auto error (false);
auto version_l = version_get (transaction_a);
Expand Down Expand Up @@ -178,11 +218,12 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size
case 11:
upgrade_v11_to_v12 (transaction_a);
case 12:
upgrade_v12_to_v13 (transaction_a, batch_size);
upgrade_v12_to_v13 (transaction_a, batch_size_a);
case 13:
upgrade_v13_to_v14 (transaction_a);
case 14:
upgrade_v14_to_v15 (transaction_a);
needs_vacuuming = true;
case 15:
break;
default:
Expand Down Expand Up @@ -499,9 +540,10 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transa
release_assert (!error || error == MDB_NOTFOUND);
}

void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transaction_a)
void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_a)
{
// Move confirmation height from account_info database to its own table
logger.always_log ("Preparing v14 to v15 upgrade...");

std::vector<std::pair<nano::account, nano::account_info>> account_infos;
upgrade_counters account_counters (count (transaction_a, accounts_v0), count (transaction_a, accounts_v1));
account_infos.reserve (account_counters.before_v0 + account_counters.before_v1);
Expand All @@ -517,10 +559,13 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
auto rep_block = block_get_v14 (transaction_a, account_info_v14.rep_block);
release_assert (rep_block != nullptr);
account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i_account.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 });
// Move confirmation height from account_info database to its own table
confirmation_height_put (transaction_a, account, account_info_v14.confirmation_height);
i_account.from_first_database ? ++account_counters.after_v0 : ++account_counters.after_v1;
}

logger.always_log ("Finished extracting confirmation height to its own database");

assert (account_counters.are_equal ());
// No longer need accounts_v1, keep v0 but clear it
mdb_drop (env.tx (transaction_a), accounts_v1, 1);
Expand All @@ -532,7 +577,7 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
mdb_put (env.tx (transaction_a), accounts, nano::mdb_val (account_account_info_pair.first), nano::mdb_val (account_info), MDB_APPEND);
}

logger.always_log ("Epoch merge upgrade. Finished accounts, now doing state blocks");
logger.always_log ("Epoch merge upgrade: Finished accounts, now doing state blocks");

account_infos.clear ();

Expand Down Expand Up @@ -567,15 +612,15 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa

// Every so often output to the log to indicate progress
constexpr auto output_cutoff = 1000000;
if (num % output_cutoff == 0)
if (num % output_cutoff == 0 && num != 0)
{
logger.always_log (boost::str (boost::format ("Database epoch merge upgrade %1% million state blocks upgraded") % (num / output_cutoff)));
}
i_state.from_first_database ? ++state_counters.after_v0 : ++state_counters.after_v1;
}

assert (state_counters.are_equal ());
logger.always_log ("Epoch merge upgrade. Finished state blocks, now doing pending blocks");
logger.always_log ("Epoch merge upgrade: Finished state blocks, now doing pending blocks");

state_blocks = state_blocks_new;

Expand Down Expand Up @@ -617,7 +662,7 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
representation = 0;
}
version_put (transaction_a, 15);
logger.always_log ("Finished epoch merge upgrade");
logger.always_log ("Finished epoch merge upgrade. Preparing vacuum...");
}

/** Takes a filepath, appends '_backup_<timestamp>' to the end (but before any extension) and saves that file in the same directory */
Expand Down
6 changes: 4 additions & 2 deletions nano/node/lmdb/lmdb.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
nano::uint128_t block_balance_computed_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const;

private:
bool do_upgrades (nano::write_transaction &, size_t);
bool do_upgrades (nano::write_transaction &, bool &, size_t);
void upgrade_v1_to_v2 (nano::write_transaction const &);
void upgrade_v2_to_v3 (nano::write_transaction const &);
void upgrade_v3_to_v4 (nano::write_transaction const &);
Expand All @@ -232,7 +232,7 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
void upgrade_v11_to_v12 (nano::write_transaction const &);
void upgrade_v12_to_v13 (nano::write_transaction &, size_t);
void upgrade_v13_to_v14 (nano::write_transaction const &);
void upgrade_v14_to_v15 (nano::write_transaction const &);
void upgrade_v14_to_v15 (nano::write_transaction &);
void open_databases (bool &, nano::transaction const &, unsigned);

int drop (nano::write_transaction const & transaction_a, tables table_a) override;
Expand All @@ -250,6 +250,8 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>

size_t count (nano::transaction const & transaction_a, tables table_a) const override;

bool vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs);

class upgrade_counters
{
public:
Expand Down
5 changes: 5 additions & 0 deletions nano/node/lmdb/lmdb_env.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
#include <nano/node/lmdb/lmdb_env.hpp>

nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
{
init (error_a, path_a, max_dbs_a, use_no_mem_init_a, map_size_a);
}

void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
{
boost::system::error_code error_mkdir, error_chmod;
if (path_a.has_parent_path ())
Expand Down
1 change: 1 addition & 0 deletions nano/node/lmdb/lmdb_env.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class mdb_env final
{
public:
mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024);
void init (bool &, boost::filesystem::path const &, int max_dbs, bool use_no_mem_init, size_t map_size = 128ULL * 1024 * 1024 * 1024);
~mdb_env ();
operator MDB_env * () const;
// clang-format off
Expand Down