fix: update seek count and loaded bytes sizes (#336)

* update seek count and loaded bytes sizes

* qfix

* qfix

costs/src/lib.rs

@@ -83,11 +83,11 @@ pub type ChildrenSizesWithIsSumTree = Option<(
 #[derive(Debug, Default, Eq, PartialEq, Clone)]
 pub struct OperationCost {
     /// How many storage_cost seeks were done.
-    pub seek_count: u16,
+    pub seek_count: u32,
     /// Storage cost of the operation.
     pub storage_cost: StorageCost,
     /// How many bytes were loaded from hard drive.
-    pub storage_loaded_bytes: u32,
+    pub storage_loaded_bytes: u64,
     /// How many times node hashing was done (for merkelized tree).
     pub hash_node_calls: u32,
 }
@@ -100,7 +100,7 @@ impl OperationCost {
 
     /// Helper function to build default `OperationCost` with different
     /// `seek_count`.
-    pub fn with_seek_count(seek_count: u16) -> Self {
+    pub fn with_seek_count(seek_count: u32) -> Self {
         OperationCost {
             seek_count,
             ..Default::default()
@@ -121,7 +121,7 @@ impl OperationCost {
 
     /// Helper function to build default `OperationCost` with different
     /// `storage_loaded_bytes`.
-    pub fn with_storage_loaded_bytes(storage_loaded_bytes: u32) -> Self {
+    pub fn with_storage_loaded_bytes(storage_loaded_bytes: u64) -> Self {
         OperationCost {
             storage_loaded_bytes,
             ..Default::default()
@@ -527,7 +527,7 @@ mod tests {
         let loaded_value = b"ayylmao";
         let costs_ctx = loaded_value.wrap_fn_cost(|x| OperationCost {
             seek_count: 1,
-            storage_loaded_bytes: x.len() as u32,
+            storage_loaded_bytes: x.len() as u64,
             ..Default::default()
         });
         assert_eq!(

grovedb/src/element/get.rs

@@ -163,7 +163,7 @@ impl Element {
                     key_ref.len() as u32,
                     value.as_ref().unwrap().len() as u32,
                     false,
-                )
+                ) as u64
             }
             Some(Element::SumItem(_, flags)) => {
                 let cost_size = SUM_ITEM_COST_SIZE;
@@ -173,7 +173,7 @@ impl Element {
                 });
                 let value_len = cost_size + flags_len;
                 cost.storage_loaded_bytes =
-                    KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false)
+                    KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) as u64
             }
             Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => {
                 let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() {
@@ -191,7 +191,7 @@ impl Element {
                         key_ref.len() as u32,
                         value_len,
                         false,
-                    )
+                    ) as u64
             }
             None => {}
         }

grovedb/src/estimated_costs/average_case_costs.rs

@@ -57,7 +57,7 @@ impl GroveDb {
                     key.max_length() as u32,
                     HASH_LENGTH as u32,
                     is_sum_tree,
-                );
+                ) as u64;
             }
         }
         *cost += S::get_storage_context_cost(path.as_vec());
@@ -434,7 +434,7 @@ impl GroveDb {
             in_parent_tree_using_sums,
         );
         cost.seek_count += 1;
-        cost.storage_loaded_bytes += value_size;
+        cost.storage_loaded_bytes += value_size as u64;
         *cost += S::get_storage_context_cost(path.as_vec());
         Ok(())
     }
@@ -561,8 +561,12 @@ impl GroveDb {
             estimated_element_size,
             in_parent_tree_using_sums,
         );
-        cost.seek_count += 1 + estimated_references_sizes.len() as u16;
-        cost.storage_loaded_bytes += value_size + estimated_references_sizes.iter().sum::<u32>();
+        cost.seek_count += 1 + estimated_references_sizes.len() as u32;
+        cost.storage_loaded_bytes += value_size as u64
+            + estimated_references_sizes
+                .iter()
+                .map(|x| *x as u64)
+                .sum::<u64>();
         *cost += S::get_storage_context_cost(path.as_vec());
         Ok(())
     }

grovedb/src/estimated_costs/worst_case_costs.rs

@@ -56,7 +56,7 @@ impl GroveDb {
                     key.max_length() as u32,
                     HASH_LENGTH as u32,
                     is_sum_tree,
-                );
+                ) as u64;
             }
         }
         *cost += S::get_storage_context_cost(path.as_vec());
@@ -407,7 +407,7 @@ impl GroveDb {
             in_parent_tree_using_sums,
         );
         cost.seek_count += 1;
-        cost.storage_loaded_bytes += value_size;
+        cost.storage_loaded_bytes += value_size as u64;
         *cost += S::get_storage_context_cost(path.as_vec());
         Ok(())
     }
@@ -498,8 +498,9 @@ impl GroveDb {
             max_element_size,
             in_parent_tree_using_sums,
         );
-        cost.seek_count += 1 + max_references_sizes.len() as u16;
-        cost.storage_loaded_bytes += value_size + max_references_sizes.iter().sum::<u32>();
+        cost.seek_count += 1 + max_references_sizes.len() as u32;
+        cost.storage_loaded_bytes +=
+            value_size as u64 + max_references_sizes.iter().map(|x| *x as u64).sum::<u64>();
         *cost += S::get_storage_context_cost(path.as_vec());
         Ok(())
     }

merk/src/estimated_costs/average_case_costs.rs

@@ -319,7 +319,7 @@ pub fn add_average_case_get_merk_node(
         not_prefixed_key_len,
         approximate_element_size,
         is_sum_tree,
-    );
+    ) as u64;
     Ok(())
 }
 
@@ -331,7 +331,7 @@ pub fn add_average_case_merk_has_value(
     estimated_element_size: u32,
 ) {
     cost.seek_count += 1;
-    cost.storage_loaded_bytes += not_prefixed_key_len + estimated_element_size;
+    cost.storage_loaded_bytes += (not_prefixed_key_len + estimated_element_size) as u64;
 }
 
 #[cfg(feature = "full")]
@@ -420,7 +420,7 @@ pub fn add_average_case_merk_propagate(
         // we can get about 1 rotation, if there are more than 2 levels
         nodes_updated += 1;
     }
-    cost.seek_count += nodes_updated as u16;
+    cost.seek_count += nodes_updated as u32;
 
     cost.hash_node_calls += nodes_updated * 2;
 
@@ -664,6 +664,6 @@ pub fn add_average_case_merk_propagate(
                 total_loaded_bytes as u32
             }
         }
-    };
+    } as u64;
     Ok(())
 }

merk/src/estimated_costs/worst_case_costs.rs

@@ -82,7 +82,8 @@ pub fn add_worst_case_get_merk_node(
     // To write a node to disk, the left link, right link and kv nodes are encoded.
     // worst case, the node has both the left and right link present.
     cost.storage_loaded_bytes +=
-        TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node);
+        TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node)
+            as u64;
     Ok(())
 }
 
@@ -94,7 +95,7 @@ pub fn add_worst_case_merk_has_value(
     max_element_size: u32,
 ) {
     cost.seek_count += 1;
-    cost.storage_loaded_bytes += not_prefixed_key_len + max_element_size;
+    cost.storage_loaded_bytes += not_prefixed_key_len as u64 + max_element_size as u64;
 }
 
 #[cfg(feature = "full")]
@@ -208,8 +209,9 @@ pub fn add_worst_case_merk_propagate(
 
     // todo: verify these numbers
     cost.storage_cost.replaced_bytes += nodes_updated * MERK_BIGGEST_VALUE_SIZE;
-    cost.storage_loaded_bytes += nodes_updated * (MERK_BIGGEST_VALUE_SIZE + MERK_BIGGEST_KEY_SIZE);
-    cost.seek_count += nodes_updated as u16;
+    cost.storage_loaded_bytes +=
+        nodes_updated as u64 * (MERK_BIGGEST_VALUE_SIZE + MERK_BIGGEST_KEY_SIZE) as u64;
+    cost.seek_count += nodes_updated;
     cost.hash_node_calls += nodes_updated * 2;
     Ok(())
 }
@@ -220,8 +222,8 @@ pub fn add_worst_case_cost_for_is_empty_tree_except(
     cost: &mut OperationCost,
     except_keys_count: u16,
 ) {
-    cost.seek_count += except_keys_count + 1;
-    cost.storage_loaded_bytes += MAX_PREFIXED_KEY_SIZE * (except_keys_count as u32 + 1);
+    cost.seek_count += except_keys_count as u32 + 1;
+    cost.storage_loaded_bytes += MAX_PREFIXED_KEY_SIZE * (except_keys_count as u64 + 1);
 }
 
 /// Add average case cost for is_empty_tree_except
@@ -231,6 +233,7 @@ pub fn add_average_case_cost_for_is_empty_tree_except(
     except_keys_count: u16,
     estimated_prefixed_key_size: u32,
 ) {
-    cost.seek_count += except_keys_count + 1;
-    cost.storage_loaded_bytes += estimated_prefixed_key_size * (except_keys_count as u32 + 1);
+    cost.seek_count += except_keys_count as u32 + 1;
+    cost.storage_loaded_bytes +=
+        estimated_prefixed_key_size as u64 * (except_keys_count as u64 + 1);
 }

merk/src/merk/defaults.rs

@@ -34,4 +34,4 @@ pub const ROOT_KEY_KEY: &[u8] = b"r";
 #[cfg(feature = "full")]
 pub const MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES: u8 = 8;
 #[cfg(feature = "full")]
-pub const MAX_PREFIXED_KEY_SIZE: u32 = 288;
+pub const MAX_PREFIXED_KEY_SIZE: u64 = 288;

storage/src/rocksdb_storage/storage.rs

@@ -280,7 +280,7 @@ impl RocksDbStorage {
                         )
                         .map(|x| x.len() as u32)
                         .unwrap_or(0);
-                        cost.storage_loaded_bytes += value_len;
+                        cost.storage_loaded_bytes += value_len as u64;
                         let key_len = key.len() as u32;
                         // todo: improve deletion
                         pending_costs.storage_cost.removed_bytes += BasicStorageRemoval(
@@ -307,7 +307,7 @@ impl RocksDbStorage {
                         )
                         .map(|x| x.len() as u32)
                         .unwrap_or(0);
-                        cost.storage_loaded_bytes += value_len;
+                        cost.storage_loaded_bytes += value_len as u64;
 
                         let key_len = key.len() as u32;
                         // todo: improve deletion
@@ -337,7 +337,7 @@ impl RocksDbStorage {
                         )
                         .map(|x| x.len() as u32)
                         .unwrap_or(0);
-                        cost.storage_loaded_bytes += value_len;
+                        cost.storage_loaded_bytes += value_len as u64;
 
                         let key_len = key.len() as u32;
                         // todo: improve deletion
@@ -367,7 +367,7 @@ impl RocksDbStorage {
                         )
                         .map(|x| x.len() as u32)
                         .unwrap_or(0);
-                        cost.storage_loaded_bytes += value_len;
+                        cost.storage_loaded_bytes += value_len as u64;
 
                         let key_len = key.len() as u32;
                         // todo: improve deletion

storage/src/rocksdb_storage/storage_context/context_no_tx.rs

@@ -209,7 +209,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -225,7 +225,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -241,7 +241,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -257,7 +257,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })

storage/src/rocksdb_storage/storage_context/context_tx.rs

@@ -240,7 +240,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -256,7 +256,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -272,7 +272,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })
@@ -288,7 +288,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> {
                     .as_ref()
                     .ok()
                     .and_then(Option::as_ref)
-                    .map(|x| x.len() as u32)
+                    .map(|x| x.len() as u64)
                     .unwrap_or(0),
                 ..Default::default()
             })

storage/src/rocksdb_storage/storage_context/raw_iterator.rs

@@ -38,7 +38,7 @@ use crate::{
 };
 
 /// 256 bytes for the key and 32 bytes for the prefix
-const MAX_PREFIXED_KEY_LENGTH: u32 = 256 + 32;
+const MAX_PREFIXED_KEY_LENGTH: u64 = 256 + 32;
 
 /// Raw iterator over prefixed storage_cost.
 pub struct PrefixedRocksDbRawIterator<I> {
@@ -91,7 +91,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
 
         let value = if self.valid().unwrap_add_cost(&mut cost) {
             self.raw_iterator.value().map(|v| {
-                cost.storage_loaded_bytes += v.len() as u32;
+                cost.storage_loaded_bytes += v.len() as u64;
                 v
             })
         } else {
@@ -109,7 +109,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
                 // Even if we truncate prefix, loaded cost should be maximum for the whole
                 // function
                 if k.starts_with(&self.prefix) {
-                    cost.storage_loaded_bytes += k.len() as u32;
+                    cost.storage_loaded_bytes += k.len() as u64;
                     Some(k.split_at(self.prefix.len()).1)
                 } else {
                     // we can think of the underlying storage layer as stacked blocks
@@ -142,7 +142,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
             .key()
             .map(|k| {
                 if k.starts_with(&self.prefix) {
-                    cost.storage_loaded_bytes += k.len() as u32;
+                    cost.storage_loaded_bytes += k.len() as u64;
                     true
                 } else {
                     // we can think of the underlying storage layer as stacked blocks
@@ -212,7 +212,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
 
         let value = if self.valid().unwrap_add_cost(&mut cost) {
             self.raw_iterator.value().map(|v| {
-                cost.storage_loaded_bytes += v.len() as u32;
+                cost.storage_loaded_bytes += v.len() as u64;
                 v
             })
         } else {
@@ -230,7 +230,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
                 // Even if we truncate prefix, loaded cost should be maximum for the whole
                 // function
                 if k.starts_with(&self.prefix) {
-                    cost.storage_loaded_bytes += k.len() as u32;
+                    cost.storage_loaded_bytes += k.len() as u64;
                     Some(k.split_at(self.prefix.len()).1)
                 } else {
                     // we can think of the underlying storage layer as stacked blocks
@@ -263,7 +263,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator<DBRawIteratorWithThreadMode<
             .key()
             .map(|k| {
                 if k.starts_with(&self.prefix) {
-                    cost.storage_loaded_bytes += k.len() as u32;
+                    cost.storage_loaded_bytes += k.len() as u64;
                     true
                 } else {
                     // we can think of the underlying storage layer as stacked blocks