From f76753156f87d2473fe488d5a4a9f62a0abe6d7f Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 18 Jun 2020 14:00:39 -0700 Subject: [PATCH] Fix partial traversals, walking with traverse_from - find and fix bug with handling sub-segments during TraversedPartialPath exception, while walking over the trie, using cached nodes, with traverse_from() in the test test_trie_walk_root_change_with_cached_traverse_from() - add example() to make sure that case ^ is tested in both tests about walking over a changed trie. - modify hypothesis inputs a bit to encourage hypothesis to find interesting scenarios like the above example. The new approach did find a relevant example once or twice, but needed low-thousands of examples. - Test the traverse_from() scenario with both approaches: emptying the cache and keeping it. Handle both scenarios properly. --- tests/test_hexary_trie_walk.py | 70 +++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/tests/test_hexary_trie_walk.py b/tests/test_hexary_trie_walk.py index 2d748887..1366c129 100644 --- a/tests/test_hexary_trie_walk.py +++ b/tests/test_hexary_trie_walk.py @@ -4,6 +4,7 @@ settings, strategies as st, ) +import pytest from trie.exceptions import ( TraversedPartialPath, @@ -14,6 +15,7 @@ from trie.fog import HexaryTrieFog, TrieFrontierCache from trie.iter import NodeIterator from trie.tools.builder import trie_from_keys +from trie.tools.strategies import trie_keys_with_extensions from trie.typing import Nibbles @@ -142,14 +144,9 @@ def test_trie_walk_backfilling_with_traverse_from(trie_keys, minimum_value_lengt @given( # starting trie keys - st.lists( - st.binary(min_size=3, max_size=3), - unique=True, - min_size=1, - max_size=1024, - ), + trie_keys_with_extensions(allow_empty_trie=False), # minimum value length (to help force trie nodes to stop embedding) - st.integers(min_value=0, max_value=32), + st.integers(min_value=1, max_value=32), # how many fog expansions to try before modifying the trie st.integers(min_value=0, max_value=10000), # all trie changes to make before the second trie walk @@ -226,6 +223,17 @@ def test_trie_walk_backfilling_with_traverse_from(trie_keys, minimum_value_lengt index_nibbles=[], index_nibbles2=[], ) +@example( + # Test that covers a TraversedPartialPath exception, to make sure the sub_segments + # are correctly generated on the simulated node of the exception + trie_keys=[b'\x01\x00\x00', b'\x01\x01\x00', b'\x00\x00'], + minimum_value_length=3, + number_explorations=2, + trie_changes=[(2, None)], + index_nibbles=[], + index_nibbles2=[], +) +@settings(max_examples=500) def test_trie_walk_root_change_with_traverse( trie_keys, minimum_value_length, @@ -242,6 +250,7 @@ def test_trie_walk_root_change_with_traverse( - Verify that all required database values were replaced (where only the nodes under the NEW trie root are required) """ + # Turn on pruning to simulate having peers lose access to old trie nodes over time node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True) number_explorations %= len(node_db) @@ -340,14 +349,9 @@ def test_trie_walk_root_change_with_traverse( @given( # starting trie keys - st.lists( - st.binary(min_size=3, max_size=3), - unique=True, - min_size=1, - max_size=1024, - ), + trie_keys_with_extensions(allow_empty_trie=False), # minimum value length (to help force trie nodes to stop embedding) - st.integers(min_value=3, max_value=32), + st.integers(min_value=1, max_value=32), # how many fog expansions to try before modifying the trie st.integers(min_value=0, max_value=10000), # all trie changes to make before the second trie walk @@ -381,8 +385,20 @@ def test_trie_walk_root_change_with_traverse( max_size=4 * 2, # one byte (two nibbles) deeper than the longest key above ), ) -@settings(max_examples=200) +@example( + # Test that covers a TraversedPartialPath exception, and make sure that the sub_segments + # and cached node are generated properly. + trie_keys=[b'\x01\x00\x00', b'\x01\x01\x00', b'\x00\x00'], + minimum_value_length=3, + number_explorations=2, + trie_changes=[(2, None)], + index_nibbles=[], + index_nibbles2=[], +) +@settings(max_examples=500) +@pytest.mark.parametrize('do_cache_reset', (True, False)) def test_trie_walk_root_change_with_cached_traverse_from( + do_cache_reset, trie_keys, minimum_value_length, number_explorations, @@ -394,6 +410,7 @@ def test_trie_walk_root_change_with_cached_traverse_from( Like test_trie_walk_root_change_with_traverse but using HexaryTrie.traverse_from when possible. """ + # Turn on pruning to simulate having peers lose access to old trie nodes over time node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True) number_explorations %= len(node_db) @@ -450,7 +467,7 @@ def test_trie_walk_root_change_with_cached_traverse_from( try: if isinstance(change, bytes): # insert! - trie_batch[change] = change + trie_batch[change] = change.rjust(minimum_value_length, b'3') expected_final_keys.add(change) else: key_index, new_value = change @@ -469,6 +486,8 @@ def test_trie_walk_root_change_with_cached_traverse_from( # Second walk index_key2 = tuple(index_nibbles2) + if do_cache_reset: + cache = TrieFrontierCache() for _ in range(100000): try: @@ -482,18 +501,25 @@ def test_trie_walk_root_change_with_cached_traverse_from( cached_node, uncached_key = cache.get(nearest_prefix) except KeyError: node = trie.traverse(nearest_prefix) + cached_node = None else: node = trie.traverse_from(cached_node, uncached_key) sub_segments = node.sub_segments except MissingTraversalNode as exc: - # Each missing node should only need to be retrieve (at most) once - node_db[exc.missing_node_hash] = missing_nodes.pop(exc.missing_node_hash) + node_hash = exc.missing_node_hash + if node_hash in missing_nodes: + # Each missing node should only need to be retrieve (at most) once + node_db[node_hash] = missing_nodes.pop(node_hash) + elif cached_node is not None: + # But, it could also be missing because of an old cached node + # Delete the bad cache and try again + cache.delete(nearest_prefix) + else: + raise AssertionError(f"Bad node hash request: {node_hash}") continue except TraversedPartialPath as exc: - sub_segments = [ - exc.nibbles_traversed + segment - for segment in exc.node.sub_segments - ] + sub_segments = exc.simulated_node.sub_segments + node = exc.simulated_node fog = fog.explore(nearest_prefix, sub_segments)