From cc94d0f666133eba7d63048e8663ec34c303be29 Mon Sep 17 00:00:00 2001 From: fominok Date: Thu, 29 Aug 2024 11:52:10 +0200 Subject: [PATCH] fix: update seek count and loaded bytes sizes (#336) * update seek count and loaded bytes sizes * qfix * qfix --- costs/src/lib.rs | 10 +++++----- grovedb/src/element/get.rs | 6 +++--- .../src/estimated_costs/average_case_costs.rs | 12 ++++++++---- .../src/estimated_costs/worst_case_costs.rs | 9 +++++---- .../src/estimated_costs/average_case_costs.rs | 8 ++++---- merk/src/estimated_costs/worst_case_costs.rs | 19 +++++++++++-------- merk/src/merk/defaults.rs | 2 +- storage/src/rocksdb_storage/storage.rs | 8 ++++---- .../storage_context/context_no_tx.rs | 8 ++++---- .../storage_context/context_tx.rs | 8 ++++---- .../storage_context/raw_iterator.rs | 14 +++++++------- 11 files changed, 56 insertions(+), 48 deletions(-) diff --git a/costs/src/lib.rs b/costs/src/lib.rs index a68670280..c3d855307 100644 --- a/costs/src/lib.rs +++ b/costs/src/lib.rs @@ -83,11 +83,11 @@ pub type ChildrenSizesWithIsSumTree = Option<( #[derive(Debug, Default, Eq, PartialEq, Clone)] pub struct OperationCost { /// How many storage_cost seeks were done. - pub seek_count: u16, + pub seek_count: u32, /// Storage cost of the operation. pub storage_cost: StorageCost, /// How many bytes were loaded from hard drive. - pub storage_loaded_bytes: u32, + pub storage_loaded_bytes: u64, /// How many times node hashing was done (for merkelized tree). pub hash_node_calls: u32, } @@ -100,7 +100,7 @@ impl OperationCost { /// Helper function to build default `OperationCost` with different /// `seek_count`. - pub fn with_seek_count(seek_count: u16) -> Self { + pub fn with_seek_count(seek_count: u32) -> Self { OperationCost { seek_count, ..Default::default() @@ -121,7 +121,7 @@ impl OperationCost { /// Helper function to build default `OperationCost` with different /// `storage_loaded_bytes`. - pub fn with_storage_loaded_bytes(storage_loaded_bytes: u32) -> Self { + pub fn with_storage_loaded_bytes(storage_loaded_bytes: u64) -> Self { OperationCost { storage_loaded_bytes, ..Default::default() @@ -527,7 +527,7 @@ mod tests { let loaded_value = b"ayylmao"; let costs_ctx = loaded_value.wrap_fn_cost(|x| OperationCost { seek_count: 1, - storage_loaded_bytes: x.len() as u32, + storage_loaded_bytes: x.len() as u64, ..Default::default() }); assert_eq!( diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 6ec625e7a..ded859d36 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -163,7 +163,7 @@ impl Element { key_ref.len() as u32, value.as_ref().unwrap().len() as u32, false, - ) + ) as u64 } Some(Element::SumItem(_, flags)) => { let cost_size = SUM_ITEM_COST_SIZE; @@ -173,7 +173,7 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) as u64 } Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => { let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() { @@ -191,7 +191,7 @@ impl Element { key_ref.len() as u32, value_len, false, - ) + ) as u64 } None => {} } diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 32d5a315f..74cfe8073 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -57,7 +57,7 @@ impl GroveDb { key.max_length() as u32, HASH_LENGTH as u32, is_sum_tree, - ); + ) as u64; } } *cost += S::get_storage_context_cost(path.as_vec()); @@ -434,7 +434,7 @@ impl GroveDb { in_parent_tree_using_sums, ); cost.seek_count += 1; - cost.storage_loaded_bytes += value_size; + cost.storage_loaded_bytes += value_size as u64; *cost += S::get_storage_context_cost(path.as_vec()); Ok(()) } @@ -561,8 +561,12 @@ impl GroveDb { estimated_element_size, in_parent_tree_using_sums, ); - cost.seek_count += 1 + estimated_references_sizes.len() as u16; - cost.storage_loaded_bytes += value_size + estimated_references_sizes.iter().sum::(); + cost.seek_count += 1 + estimated_references_sizes.len() as u32; + cost.storage_loaded_bytes += value_size as u64 + + estimated_references_sizes + .iter() + .map(|x| *x as u64) + .sum::(); *cost += S::get_storage_context_cost(path.as_vec()); Ok(()) } diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 2daf18b66..4c409b999 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -56,7 +56,7 @@ impl GroveDb { key.max_length() as u32, HASH_LENGTH as u32, is_sum_tree, - ); + ) as u64; } } *cost += S::get_storage_context_cost(path.as_vec()); @@ -407,7 +407,7 @@ impl GroveDb { in_parent_tree_using_sums, ); cost.seek_count += 1; - cost.storage_loaded_bytes += value_size; + cost.storage_loaded_bytes += value_size as u64; *cost += S::get_storage_context_cost(path.as_vec()); Ok(()) } @@ -498,8 +498,9 @@ impl GroveDb { max_element_size, in_parent_tree_using_sums, ); - cost.seek_count += 1 + max_references_sizes.len() as u16; - cost.storage_loaded_bytes += value_size + max_references_sizes.iter().sum::(); + cost.seek_count += 1 + max_references_sizes.len() as u32; + cost.storage_loaded_bytes += + value_size as u64 + max_references_sizes.iter().map(|x| *x as u64).sum::(); *cost += S::get_storage_context_cost(path.as_vec()); Ok(()) } diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 12f8c2c92..fc08ab45d 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -319,7 +319,7 @@ pub fn add_average_case_get_merk_node( not_prefixed_key_len, approximate_element_size, is_sum_tree, - ); + ) as u64; Ok(()) } @@ -331,7 +331,7 @@ pub fn add_average_case_merk_has_value( estimated_element_size: u32, ) { cost.seek_count += 1; - cost.storage_loaded_bytes += not_prefixed_key_len + estimated_element_size; + cost.storage_loaded_bytes += (not_prefixed_key_len + estimated_element_size) as u64; } #[cfg(feature = "full")] @@ -420,7 +420,7 @@ pub fn add_average_case_merk_propagate( // we can get about 1 rotation, if there are more than 2 levels nodes_updated += 1; } - cost.seek_count += nodes_updated as u16; + cost.seek_count += nodes_updated as u32; cost.hash_node_calls += nodes_updated * 2; @@ -664,6 +664,6 @@ pub fn add_average_case_merk_propagate( total_loaded_bytes as u32 } } - }; + } as u64; Ok(()) } diff --git a/merk/src/estimated_costs/worst_case_costs.rs b/merk/src/estimated_costs/worst_case_costs.rs index f4623c8dd..9ae6c2b3c 100644 --- a/merk/src/estimated_costs/worst_case_costs.rs +++ b/merk/src/estimated_costs/worst_case_costs.rs @@ -82,7 +82,8 @@ pub fn add_worst_case_get_merk_node( // To write a node to disk, the left link, right link and kv nodes are encoded. // worst case, the node has both the left and right link present. cost.storage_loaded_bytes += - TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node); + TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node) + as u64; Ok(()) } @@ -94,7 +95,7 @@ pub fn add_worst_case_merk_has_value( max_element_size: u32, ) { cost.seek_count += 1; - cost.storage_loaded_bytes += not_prefixed_key_len + max_element_size; + cost.storage_loaded_bytes += not_prefixed_key_len as u64 + max_element_size as u64; } #[cfg(feature = "full")] @@ -208,8 +209,9 @@ pub fn add_worst_case_merk_propagate( // todo: verify these numbers cost.storage_cost.replaced_bytes += nodes_updated * MERK_BIGGEST_VALUE_SIZE; - cost.storage_loaded_bytes += nodes_updated * (MERK_BIGGEST_VALUE_SIZE + MERK_BIGGEST_KEY_SIZE); - cost.seek_count += nodes_updated as u16; + cost.storage_loaded_bytes += + nodes_updated as u64 * (MERK_BIGGEST_VALUE_SIZE + MERK_BIGGEST_KEY_SIZE) as u64; + cost.seek_count += nodes_updated; cost.hash_node_calls += nodes_updated * 2; Ok(()) } @@ -220,8 +222,8 @@ pub fn add_worst_case_cost_for_is_empty_tree_except( cost: &mut OperationCost, except_keys_count: u16, ) { - cost.seek_count += except_keys_count + 1; - cost.storage_loaded_bytes += MAX_PREFIXED_KEY_SIZE * (except_keys_count as u32 + 1); + cost.seek_count += except_keys_count as u32 + 1; + cost.storage_loaded_bytes += MAX_PREFIXED_KEY_SIZE * (except_keys_count as u64 + 1); } /// Add average case cost for is_empty_tree_except @@ -231,6 +233,7 @@ pub fn add_average_case_cost_for_is_empty_tree_except( except_keys_count: u16, estimated_prefixed_key_size: u32, ) { - cost.seek_count += except_keys_count + 1; - cost.storage_loaded_bytes += estimated_prefixed_key_size * (except_keys_count as u32 + 1); + cost.seek_count += except_keys_count as u32 + 1; + cost.storage_loaded_bytes += + estimated_prefixed_key_size as u64 * (except_keys_count as u64 + 1); } diff --git a/merk/src/merk/defaults.rs b/merk/src/merk/defaults.rs index c5c453c31..87734e9b0 100644 --- a/merk/src/merk/defaults.rs +++ b/merk/src/merk/defaults.rs @@ -34,4 +34,4 @@ pub const ROOT_KEY_KEY: &[u8] = b"r"; #[cfg(feature = "full")] pub const MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES: u8 = 8; #[cfg(feature = "full")] -pub const MAX_PREFIXED_KEY_SIZE: u32 = 288; +pub const MAX_PREFIXED_KEY_SIZE: u64 = 288; diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index a396b75fe..8a91d4f47 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -280,7 +280,7 @@ impl RocksDbStorage { ) .map(|x| x.len() as u32) .unwrap_or(0); - cost.storage_loaded_bytes += value_len; + cost.storage_loaded_bytes += value_len as u64; let key_len = key.len() as u32; // todo: improve deletion pending_costs.storage_cost.removed_bytes += BasicStorageRemoval( @@ -307,7 +307,7 @@ impl RocksDbStorage { ) .map(|x| x.len() as u32) .unwrap_or(0); - cost.storage_loaded_bytes += value_len; + cost.storage_loaded_bytes += value_len as u64; let key_len = key.len() as u32; // todo: improve deletion @@ -337,7 +337,7 @@ impl RocksDbStorage { ) .map(|x| x.len() as u32) .unwrap_or(0); - cost.storage_loaded_bytes += value_len; + cost.storage_loaded_bytes += value_len as u64; let key_len = key.len() as u32; // todo: improve deletion @@ -367,7 +367,7 @@ impl RocksDbStorage { ) .map(|x| x.len() as u32) .unwrap_or(0); - cost.storage_loaded_bytes += value_len; + cost.storage_loaded_bytes += value_len as u64; let key_len = key.len() as u32; // todo: improve deletion diff --git a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs b/storage/src/rocksdb_storage/storage_context/context_no_tx.rs index fd639a5a6..80ad01499 100644 --- a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_no_tx.rs @@ -209,7 +209,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -225,7 +225,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -241,7 +241,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -257,7 +257,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) diff --git a/storage/src/rocksdb_storage/storage_context/context_tx.rs b/storage/src/rocksdb_storage/storage_context/context_tx.rs index d5a480c38..0272174b7 100644 --- a/storage/src/rocksdb_storage/storage_context/context_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_tx.rs @@ -240,7 +240,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -256,7 +256,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -272,7 +272,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) @@ -288,7 +288,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .as_ref() .ok() .and_then(Option::as_ref) - .map(|x| x.len() as u32) + .map(|x| x.len() as u64) .unwrap_or(0), ..Default::default() }) diff --git a/storage/src/rocksdb_storage/storage_context/raw_iterator.rs b/storage/src/rocksdb_storage/storage_context/raw_iterator.rs index a9d6cf4fe..7cc2d1fef 100644 --- a/storage/src/rocksdb_storage/storage_context/raw_iterator.rs +++ b/storage/src/rocksdb_storage/storage_context/raw_iterator.rs @@ -38,7 +38,7 @@ use crate::{ }; /// 256 bytes for the key and 32 bytes for the prefix -const MAX_PREFIXED_KEY_LENGTH: u32 = 256 + 32; +const MAX_PREFIXED_KEY_LENGTH: u64 = 256 + 32; /// Raw iterator over prefixed storage_cost. pub struct PrefixedRocksDbRawIterator { @@ -91,7 +91,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator