separate participation cache by epoch and add it to the beacon state

beacon_node/http_api/src/attestation_performance.rs

@@ -3,11 +3,11 @@ use eth2::lighthouse::{
     AttestationPerformance, AttestationPerformanceQuery, AttestationPerformanceStatistics,
 };
 use state_processing::{
-    per_epoch_processing::altair::participation_cache::Error as ParticipationCacheError,
-    per_epoch_processing::EpochProcessingSummary, BlockReplayError, BlockReplayer,
+    BlockReplayer, BlockReplayError, per_epoch_processing::EpochProcessingSummary,
 };
 use std::sync::Arc;
 use types::{BeaconState, BeaconStateError, EthSpec, Hash256, SignedBeaconBlock};
+use types::beacon_state::participation_cache::Error as ParticipationCacheError;
 use warp_utils::reject::{beacon_chain_error, custom_bad_request, custom_server_error};
 
 const MAX_REQUEST_RANGE_EPOCHS: usize = 100;

beacon_node/http_api/src/validator_inclusion.rs

@@ -5,10 +5,11 @@ use eth2::{
     types::ValidatorId,
 };
 use state_processing::per_epoch_processing::{
-    altair::participation_cache::Error as ParticipationCacheError, process_epoch,
     EpochProcessingSummary,
+    process_epoch,
 };
 use types::{BeaconState, ChainSpec, Epoch, EthSpec};
+use types::beacon_state::participation_cache::Error as ParticipationCacheError;
 
 /// Returns the state in the last slot of `epoch`.
 fn end_of_epoch_state<T: BeaconChainTypes>(

consensus/state_processing/src/lib.rs

@@ -26,17 +26,17 @@ pub mod state_advance;
 pub mod upgrade;
 pub mod verify_operation;
 
-pub use block_replayer::{BlockReplayError, BlockReplayer, StateRootStrategy};
+pub use block_replayer::{BlockReplayer, BlockReplayError, StateRootStrategy};
 pub use genesis::{
     eth2_genesis_time, initialize_beacon_state_from_eth1, is_valid_genesis_state,
     process_activations,
 };
 pub use per_block_processing::{
-    block_signature_verifier, errors::BlockProcessingError, per_block_processing, signature_sets,
-    BlockSignatureStrategy, BlockSignatureVerifier, VerifyBlockRoot, VerifySignatures,
+    altair::sync_committee, block_signature_verifier, BlockSignatureStrategy, BlockSignatureVerifier,
+    errors::BlockProcessingError, per_block_processing, signature_sets, VerifyBlockRoot, VerifySignatures
 };
 pub use per_epoch_processing::{
-    errors::EpochProcessingError, process_epoch as per_epoch_processing,
+    errors::EpochProcessingError, process_epoch as per_epoch_processing
 };
-pub use per_slot_processing::{per_slot_processing, Error as SlotProcessingError};
+pub use per_slot_processing::{Error as SlotProcessingError, per_slot_processing};
 pub use verify_operation::{SigVerifiedOp, VerifyOperation};

consensus/state_processing/src/per_epoch_processing/altair.rs

@@ -1,20 +1,20 @@
-use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error};
+use super::{EpochProcessingSummary, Error, process_registry_updates, process_slashings};
 use crate::per_epoch_processing::{
     effective_balance_updates::process_effective_balance_updates,
     historical_roots_update::process_historical_roots_update,
     resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset},
 };
 pub use inactivity_updates::process_inactivity_updates;
 pub use justification_and_finalization::process_justification_and_finalization;
-pub use participation_cache::ParticipationCache;
+pub use types::beacon_state::participation_cache::ParticipationCache;
 pub use participation_flag_updates::process_participation_flag_updates;
 pub use rewards_and_penalties::process_rewards_and_penalties;
 pub use sync_committee_updates::process_sync_committee_updates;
 use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch};
+use types::beacon_state::participation_cache::CurrentEpochParticipationCache;
 
 pub mod inactivity_updates;
 pub mod justification_and_finalization;
-pub mod participation_cache;
 pub mod participation_flag_updates;
 pub mod rewards_and_penalties;
 pub mod sync_committee_updates;
@@ -23,17 +23,12 @@ pub fn process_epoch<T: EthSpec>(
     state: &mut BeaconState<T>,
     spec: &ChainSpec,
 ) -> Result<EpochProcessingSummary<T>, Error> {
-    // Ensure the committee caches are built.
-    state.build_committee_cache(RelativeEpoch::Previous, spec)?;
-    state.build_committee_cache(RelativeEpoch::Current, spec)?;
+
     state.build_committee_cache(RelativeEpoch::Next, spec)?;
 
-    // Pre-compute participating indices and total balances.
-    let participation_cache = ParticipationCache::new(state, spec)?;
-    let sync_committee = state.current_sync_committee()?.clone();
+    let participation_cache = process_justifiable(state, spec)?;
 
-    // Justification and finalization.
-    process_justification_and_finalization(state, &participation_cache)?;
+    let sync_committee = state.current_sync_committee()?.clone();
 
     process_inactivity_updates(state, &participation_cache, spec)?;
 
@@ -78,3 +73,21 @@ pub fn process_epoch<T: EthSpec>(
         sync_committee,
     })
 }
+
+pub fn process_justifiable<T: EthSpec>(state: &mut BeaconState<T>, spec: &ChainSpec) -> Result<ParticipationCache, Error> {
+// Ensure the committee caches are built.
+    state.build_committee_cache(RelativeEpoch::Previous, spec)?;
+    state.build_committee_cache(RelativeEpoch::Current, spec)?;
+
+    // Pre-compute participating indices and total balances.
+    let prev_participation_cache = state.get_previous_epoch_participation_cache(spec)?;
+
+    let current_participation_cache = CurrentEpochParticipationCache::new(state, spec)?;
+
+    let participation_cache = ParticipationCache::new(prev_participation_cache, current_participation_cache);
+
+    // Justification and finalization.
+    process_justification_and_finalization(state, &participation_cache)?;
+
+    Ok(participation_cache)
+}

consensus/state_processing/src/per_epoch_processing/altair/justification_and_finalization.rs

@@ -4,6 +4,7 @@ use crate::per_epoch_processing::Error;
 use safe_arith::SafeArith;
 use types::consts::altair::TIMELY_TARGET_FLAG_INDEX;
 use types::{BeaconState, EthSpec};
+use types::beacon_state::participation_cache::CurrentEpochParticipationCache;
 
 /// Update the justified and finalized checkpoints for matching target attestations.
 pub fn process_justification_and_finalization<T: EthSpec>(

consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs

@@ -8,6 +8,7 @@ use types::{BeaconState, ChainSpec, EthSpec};
 
 use crate::common::{altair::get_base_reward, decrease_balance, increase_balance};
 use crate::per_epoch_processing::{Delta, Error};
+use types::beacon_state::participation_cache::CurrentEpochParticipationCache;
 
 /// Apply attester and proposer rewards.
 ///

consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs

@@ -1,10 +1,11 @@
 use super::{
-    altair::{participation_cache::Error as ParticipationCacheError, ParticipationCache},
-    base::{validator_statuses::InclusionInfo, TotalBalances, ValidatorStatus},
+    altair::ParticipationCache,
+    base::{TotalBalances, validator_statuses::InclusionInfo, ValidatorStatus},
 };
 use crate::metrics;
 use std::sync::Arc;
 use types::{EthSpec, SyncCommittee};
+use types::beacon_state::participation_cache::Error as ParticipationCacheError;
 
 /// Provides a summary of validator participation during the epoch.
 #[derive(PartialEq, Debug)]

consensus/state_processing/src/per_epoch_processing/errors.rs

@@ -1,4 +1,4 @@
-use crate::per_epoch_processing::altair::participation_cache::Error as ParticipationCacheError;
+use types::beacon_state::participation_cache::Error as ParticipationCacheError;
 use types::{BeaconStateError, InconsistentFork};
 
 #[derive(Debug, PartialEq)]

consensus/state_processing/src/upgrade/altair.rs

@@ -105,6 +105,7 @@ pub fn upgrade_to_altair<E: EthSpec>(
         pubkey_cache: mem::take(&mut pre.pubkey_cache),
         exit_cache: mem::take(&mut pre.exit_cache),
         tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
+        previous_epoch_participation_cache: mem::take(&mut pre.previous_epoch_participation_cache),
     });
 
     // Fill in previous epoch participation from the pre state's pending attestations.

consensus/state_processing/src/upgrade/merge.rs

@@ -64,6 +64,7 @@ pub fn upgrade_to_bellatrix<E: EthSpec>(
         pubkey_cache: mem::take(&mut pre.pubkey_cache),
         exit_cache: mem::take(&mut pre.exit_cache),
         tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
+        previous_epoch_participation_cache: mem::take(&mut pre.previous_epoch_participation_cache),
     });
 
     *pre_state = post;

consensus/types/src/beacon_state.rs

@@ -10,20 +10,21 @@ use int_to_bytes::{int_to_bytes4, int_to_bytes8};
 use pubkey_cache::PubkeyCache;
 use safe_arith::{ArithError, SafeArith};
 use serde_derive::{Deserialize, Serialize};
-use ssz::{ssz_encode, Decode, DecodeError, Encode};
+use ssz::{Decode, DecodeError, Encode, ssz_encode};
 use ssz_derive::{Decode, Encode};
-use ssz_types::{typenum::Unsigned, BitVector, FixedVector};
+use ssz_types::{BitVector, FixedVector, typenum::Unsigned};
 use std::convert::TryInto;
 use std::{fmt, mem, sync::Arc};
 use superstruct::superstruct;
 use swap_or_not_shuffle::compute_shuffled_index;
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
+use participation_cache::PreviousParticipationCache;
 
 pub use self::committee_cache::{
-    compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
-    CommitteeCache,
+    CommitteeCache, compute_committee_index_in_epoch, compute_committee_range_in_epoch,
+    epoch_committee_count,
 };
 pub use clone_config::CloneConfig;
 pub use eth_spec::*;
@@ -38,6 +39,7 @@ mod iter;
 mod pubkey_cache;
 mod tests;
 mod tree_hash_cache;
+pub mod participation_cache;
 
 pub const CACHED_EPOCHS: usize = 3;
 const MAX_RANDOM_BYTE: u64 = (1 << 8) - 1;
@@ -311,6 +313,12 @@ where
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]
     pub tree_hash_cache: BeaconTreeHashCache<T>,
+    #[serde(skip_serializing, skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
+    #[tree_hash(skip_hashing)]
+    #[test_random(default)]
+    #[derivative(Clone(clone_with = "clone_default"))]
+    pub previous_epoch_participation_cache: PreviousParticipationCache,
 }
 
 impl<T: EthSpec> Clone for BeaconState<T> {
@@ -376,6 +384,7 @@ impl<T: EthSpec> BeaconState<T> {
             pubkey_cache: PubkeyCache::default(),
             exit_cache: ExitCache::default(),
             tree_hash_cache: <_>::default(),
+            previous_epoch_participation_cache: <_>::default(),
         })
     }
 
@@ -1353,6 +1362,16 @@ impl<T: EthSpec> BeaconState<T> {
         Ok(())
     }
 
+    pub fn get_previous_epoch_participation_cache(&mut self, spec: &ChainSpec) -> Result<&PreviousParticipationCache, BeaconStateError> {
+        if self.previous_epoch_participation_cache().previous_epoch() == self.slot().epoch(T::slots_per_epoch()) - 1 {
+            Ok(&self.previous_epoch_participation_cache())
+        } else {
+            //rebuild cache
+            *self.previous_epoch_participation_cache_mut() = PreviousParticipationCache::new(self, spec)?;
+            Ok(&self.previous_epoch_participation_cache())
+        }
+    }
+
     /// Returns `true` if the committee cache for `relative_epoch` is built and ready to use.
     pub fn committee_cache_is_initialized(&self, relative_epoch: RelativeEpoch) -> bool {
         let i = Self::committee_cache_index(relative_epoch);