From 88aeba09f6edd9ada8a2ec7fc94e8aa90fb9ecdf Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 12 Oct 2019 12:05:08 +0900 Subject: [PATCH 001/111] Added new shards --- specs/core/1_new_shards.md | 147 +++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 specs/core/1_new_shards.md diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md new file mode 100644 index 0000000000..dec84dfe11 --- /dev/null +++ b/specs/core/1_new_shards.md @@ -0,0 +1,147 @@ +# Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +- [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains) + - [Table of contents](#table-of-contents) + - [Introduction](#introduction) + - [Configuration](#configuration) + - [Misc](#misc) + - [Containers](#containers) + - [Beacon Chain Changes](#beacon-chain-changes) + - [New state variables](#new-state-variables) + + + +## Introduction + +This document describes the shard transition function (data layer only) and the shard fork choice rule as part of Phase 1 of Ethereum 2.0. + +## Configuration + +### Misc + +| Name | Value | +| - | - | +| `MAX_SHARDS` | `2**10` (= 1024) | +| `ACTIVE_SHARDS` | `2**6` (= 64) | +| `SHARD_ROOT_HISTORY_LENGTH` | `2**15` (= 32,768) | +| `MAX_CATCHUP` | `2**3` (= 8) | + +## Containers + +### `AttestationData` + +```python +class AttestationData(Container): + # Slot + slot: Slot + # Shard + shard: shard + # LMD GHOST vote + beacon_block_root: Hash + # FFG vote + source: Checkpoint + target: Checkpoint + # Shard data roots + shard_data_roots: List[Hash, MAX_CATCHUP] + # Intermediate state roots + shard_state_roots: List[Hash, MAX_CATCHUP] +``` + +### `Attestation` + +```python +class Attestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP] + signature: BLSSignature +``` + +## Beacon Chain Changes + +### New state variables + +``` + shard_state_roots: Vector[Hash, MAX_SHARDS] + shard_next_slot: Vector[Slot, MAX_SHARDS] +``` + +### Attestation processing + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert shard < ACTIVE_SHARDS + + # Signature check + committee = get_crosslink_committee(state, get_current_epoch(state), data.shard) + for bits in attestation.custody_bits + [attestation.aggregation_bits]: + assert bits == len(committee) + # Check signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) + + # Type 1: on-time attestations + if data.custody_bits != []: + # Correct start slot + assert data.slot == state.shard_next_slot[data.shard] + # Correct data root count + assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, MAX_CATCHUP) + # Correct parent block root + assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) + # Apply + online_indices = get_online_indices(state) + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits).intersection(get_online_indices) + if get_total_balance(state, attesting_indices) * 3 >= get_total_balance(state, online_indices) * 2: + state.shard_state_roots[data.shard] = data.shard_state_roots[-1] + state.shard_next_slot[data.shard] += len(data.shard_data_roots) + + # Type 2: delayed attestations + else: + assert slot_to_epoch(data.slot) in (get_current_epoch(state), get_previous_epoch(state)) + assert len(data.shard_data_roots) == len(data.intermediate_state_roots) == 0 + + pending_attestation = PendingAttestation( + slot=data.slot, + shard=data.shard, + aggregation_bits=attestation.aggregation_bits, + inclusion_delay=state.slot - attestation_slot, + proposer_index=get_beacon_proposer_index(state), + ) + + if data.target.epoch == get_current_epoch(state): + assert data.source == state.current_justified_checkpoint + state.current_epoch_attestations.append(pending_attestation) + else: + assert data.source == state.previous_justified_checkpoint + state.previous_epoch_attestations.append(pending_attestation) +``` + +### Fraud proofs + +TODO. The intent is to have a single universal fraud proof type, which contains (i) an on-time attestation on shard `s` signing a set of `data_roots`, (ii) an index `i` of a particular data root to focus on, (iii) the full contents of the i'th data, (iii) a Merkle proof to the `shard_state_roots` in the parent block the attestation is referencing, and which then verifies that one of the two conditions is false: + +* `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` +* `execute_state_transition(slot, shard, attestation.shard_state_roots[i-1], parent.shard_state_roots, block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s]`) + +For phase 1, we will use a simple state transition function: + +* Check that `data[:32] == prev_state_root` +* Check that `bls_verify(get_shard_proposer(state, slot, shard), hash_tree_root(data[-96:]), BLSSignature(data[-96:]), BLOCK_SIGNATURE_DOMAIN)` +* Output the new state root: `hash_tree_root(prev_state_root, other_prev_state_roots, data)` + +### Honest persistent committee member behavior + +Suppose you are a persistent committee member on shard `i` at slot `s`. Suppose `state.shard_next_slots[i] = s-1` ("the happy case"). In this case, you look for a valid proposal that satisfies the checks in the state transition function above, and if you see such a proposal `data` with post-state `post_state`, make an attestation with `shard_data_roots = [hash_tree_root(data)]` and `shard_state_roots = [post_state]`. If you do not find such a proposal, make an attestation using the "default empty proposal", `data = prev_state_root + b'\x00' * 96`. + +Now suppose `state.shard_next_slots[i] = s-k` for `k>1`. Then, initialize `data = []`, `states = []`, `state = state.shard_state_roots[i]`. For `slot in (state.shard_next_slot, min(state.shard_next_slot + MAX_CATCHUP, s))`, do: + +* Look for all valid proposals for `slot` whose first 32 bytes equal to `state`. If there are none, add a default empty proposal to `data`. If there is one such proposal `p`, add `p` to `data`. If there is more than one, select the one with the largest number of total attestations supporting it or its descendants, and add it to `data`. +* Set `state` to the state after processing the proposal just added to `data`; append it to `states` + +Make an attestation using `shard_data_roots = data` and `shard_state_roots = states`. From f6be6b2b8aa47ef4f9c696e5bed62f7b6d2512f6 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 12 Oct 2019 23:59:51 +0900 Subject: [PATCH 002/111] Added a few things --- specs/core/1_new_shards.md | 65 ++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index dec84dfe11..2066aac8de 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -12,8 +12,13 @@ - [Configuration](#configuration) - [Misc](#misc) - [Containers](#containers) + - [Helpers](#helpers) - [Beacon Chain Changes](#beacon-chain-changes) - [New state variables](#new-state-variables) + - [Attestation processing](#attestation-processing) + - [Epoch transition](#epoch-transition) + - [Fraud proofs](#fraud-proofs) + - [Honest persistent committee member behavior](#honest-persistent-committee-member-behavior) @@ -30,7 +35,8 @@ This document describes the shard transition function (data layer only) and the | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | | `SHARD_ROOT_HISTORY_LENGTH` | `2**15` (= 32,768) | -| `MAX_CATCHUP` | `2**3` (= 8) | +| `MAX_CATCHUP` | `2**5` (= 32) | +| `ONLINE_PERIOD` | `2**3` (= 8) | ## Containers @@ -40,8 +46,6 @@ This document describes the shard transition function (data layer only) and the class AttestationData(Container): # Slot slot: Slot - # Shard - shard: shard # LMD GHOST vote beacon_block_root: Hash # FFG vote @@ -51,6 +55,8 @@ class AttestationData(Container): shard_data_roots: List[Hash, MAX_CATCHUP] # Intermediate state roots shard_state_roots: List[Hash, MAX_CATCHUP] + # Index + index: uint64 ``` ### `Attestation` @@ -63,13 +69,31 @@ class Attestation(Container): signature: BLSSignature ``` +## Helpers + +### `get_online_validators` + +```python +def get_online_indices(state: BeaconState) -> Set[ValidatorIndex]: + active_validators = get_active_validator_indices(state, get_current_epoch(state)) + return set([i for i in active_validators if state.online_countdown[i] != 0]) +``` + +### `get_shard_state_root` + +```python +def get_shard_state_root(state: BeaconState, shard: Shard) -> Hash: + return state.shard_state_roots[shard][-1] +``` + ## Beacon Chain Changes ### New state variables ``` - shard_state_roots: Vector[Hash, MAX_SHARDS] + shard_state_roots: Vector[List[Hash, MAX_CATCHUP], MAX_SHARDS] shard_next_slot: Vector[Slot, MAX_SHARDS] + online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] ``` ### Attestation processing @@ -77,38 +101,43 @@ class Attestation(Container): ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data - assert shard < ACTIVE_SHARDS + assert data.index < ACTIVE_SHARDS + shard = (data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS # Signature check - committee = get_crosslink_committee(state, get_current_epoch(state), data.shard) + committee = get_crosslink_committee(state, get_current_epoch(state), shard) for bits in attestation.custody_bits + [attestation.aggregation_bits]: assert bits == len(committee) # Check signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) + # Get attesting indices + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) # Type 1: on-time attestations if data.custody_bits != []: # Correct start slot - assert data.slot == state.shard_next_slot[data.shard] + assert data.slot == state.shard_next_slot[shard] # Correct data root count assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, MAX_CATCHUP) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Apply online_indices = get_online_indices(state) - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits).intersection(get_online_indices) - if get_total_balance(state, attesting_indices) * 3 >= get_total_balance(state, online_indices) * 2: - state.shard_state_roots[data.shard] = data.shard_state_roots[-1] - state.shard_next_slot[data.shard] += len(data.shard_data_roots) + if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: + state.shard_state_roots[shard] = data.shard_state_roots + state.shard_next_slot[shard] += len(data.shard_data_roots) # Type 2: delayed attestations else: assert slot_to_epoch(data.slot) in (get_current_epoch(state), get_previous_epoch(state)) assert len(data.shard_data_roots) == len(data.intermediate_state_roots) == 0 + for index in attesting_indices: + online_countdown[index] = ONLINE_PERIOD + pending_attestation = PendingAttestation( slot=data.slot, - shard=data.shard, + shard=shard, aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), @@ -122,12 +151,20 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` +### Epoch transition + +```python +for index in range(len(state.validators)): + if state.online_countdown[index] != 0: + state.online_countdown[index] = state.online_countdown[index] - 1 +``` + ### Fraud proofs TODO. The intent is to have a single universal fraud proof type, which contains (i) an on-time attestation on shard `s` signing a set of `data_roots`, (ii) an index `i` of a particular data root to focus on, (iii) the full contents of the i'th data, (iii) a Merkle proof to the `shard_state_roots` in the parent block the attestation is referencing, and which then verifies that one of the two conditions is false: * `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -* `execute_state_transition(slot, shard, attestation.shard_state_roots[i-1], parent.shard_state_roots, block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s]`) +* `execute_state_transition(slot, shard, attestation.shard_state_roots[i-1], parent.shard_state_roots, block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s][-1]`) For phase 1, we will use a simple state transition function: @@ -135,7 +172,7 @@ For phase 1, we will use a simple state transition function: * Check that `bls_verify(get_shard_proposer(state, slot, shard), hash_tree_root(data[-96:]), BLSSignature(data[-96:]), BLOCK_SIGNATURE_DOMAIN)` * Output the new state root: `hash_tree_root(prev_state_root, other_prev_state_roots, data)` -### Honest persistent committee member behavior +## Honest persistent committee member behavior Suppose you are a persistent committee member on shard `i` at slot `s`. Suppose `state.shard_next_slots[i] = s-1` ("the happy case"). In this case, you look for a valid proposal that satisfies the checks in the state transition function above, and if you see such a proposal `data` with post-state `post_state`, make an attestation with `shard_data_roots = [hash_tree_root(data)]` and `shard_state_roots = [post_state]`. If you do not find such a proposal, make an attestation using the "default empty proposal", `data = prev_state_root + b'\x00' * 96`. From 25db268bfb815709667e15ff6d401e44b143eee5 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 13 Oct 2019 15:52:51 +0900 Subject: [PATCH 003/111] Added a few more things --- specs/core/1_new_shards.md | 95 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 2066aac8de..cf644b4c0c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -15,7 +15,9 @@ - [Helpers](#helpers) - [Beacon Chain Changes](#beacon-chain-changes) - [New state variables](#new-state-variables) + - [New block data structures](#new-block-data-structures) - [Attestation processing](#attestation-processing) + - [Light client signature processing)(#light-client-signature-processing) - [Epoch transition](#epoch-transition) - [Fraud proofs](#fraud-proofs) - [Honest persistent committee member behavior](#honest-persistent-committee-member-behavior) @@ -30,13 +32,15 @@ This document describes the shard transition function (data layer only) and the ### Misc -| Name | Value | -| - | - | +| Name | Value | Unit | Duration | +| - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | | `SHARD_ROOT_HISTORY_LENGTH` | `2**15` (= 32,768) | -| `MAX_CATCHUP` | `2**5` (= 32) | -| `ONLINE_PERIOD` | `2**3` (= 8) | +| `MAX_CATCHUP` | `2**5` (= 32) | slots | 3.2 min | +| `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | +| `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | ## Containers @@ -69,6 +73,14 @@ class Attestation(Container): signature: BLSSignature ``` +### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + ## Helpers ### `get_online_validators` @@ -86,6 +98,44 @@ def get_shard_state_root(state: BeaconState, shard: Shard) -> Hash: return state.shard_state_roots[shard][-1] ``` +### `pack_compact_validator` + +```python +def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: + """ + Creates a compact validator object representing index, slashed status, and compressed balance. + Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with + the unpacking function. + """ + return (index << 16) + (slashed << 15) + balance_in_increments +``` + +### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: + """ + Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1) +``` + +### `committee_to_compact_committee` + +```python +def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + """ + Given a state and a list of validator indices, outputs the CompactCommittee representing them. + """ + validators = [state.validators[i] for i in committee] + compact_validators = [ + pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) + for i, v in zip(committee, validators) + ] + pubkeys = [v.pubkey for v in validators] + return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) +``` + ## Beacon Chain Changes ### New state variables @@ -94,6 +144,15 @@ def get_shard_state_root(state: BeaconState, shard: Shard) -> Hash: shard_state_roots: Vector[List[Hash, MAX_CATCHUP], MAX_SHARDS] shard_next_slot: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] + current_light_committee: CompactCommittee + next_light_committee: CompactCommittee +``` + +### New block data structures + +``` + light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_signature: BLSSignature ``` ### Attestation processing @@ -151,12 +210,40 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` +### Light client processing + +```python +signer_validators = [] +signer_keys = [] +for i, bit in enumerate(block.light_client_signature_bitfield): + if bit: + signer_keys.append(state.current_light_committee.pubkeys[i]) + index, _, _ = unpack_compact_validator(state.current_light_committee.compact_validators[i]) + signer_validators.append(index) + +assert bls_verify( + pubkey=bls_aggregate_pubkeys(signer_keys), + message_hash=get_block_root_at_slot(state, state.slot - 1), + signature=block.light_client_signature, + domain=DOMAIN_LIGHT_CLIENT +) +``` + ### Epoch transition ```python +# Slowly remove validators from the "online" set if they do not show up for index in range(len(state.validators)): if state.online_countdown[index] != 0: state.online_countdown[index] = state.online_countdown[index] - 1 + +# Update light client committees +if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: + state.current_light_committee = state.next_light_committee + seed = get_seed(state, get_current_epoch(state), DOMAIN_LIGHT_CLIENT) + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + committee = [active_indices[compute_shuffled_index(ValidatorIndex(i), len(active_indices), seed)] for i in range(LIGHT_CLIENT_COMMITTEE_SIZE)] + state.next_light_committee = committee_to_compact_committee(state, committee) ``` ### Fraud proofs From 26df98bf61b0d07d60263dbe339e80d93c3e6098 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 13 Oct 2019 17:11:29 +0900 Subject: [PATCH 004/111] MAX_CATCHUP -> MAX_CATCHUP_RATIO --- specs/core/1_new_shards.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index cf644b4c0c..2d199a6409 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -37,7 +37,7 @@ This document describes the shard transition function (data layer only) and the | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | | `SHARD_ROOT_HISTORY_LENGTH` | `2**15` (= 32,768) | -| `MAX_CATCHUP` | `2**5` (= 32) | slots | 3.2 min | +| `MAX_CATCHUP_RATIO` | `2**2` (= 4) | | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | @@ -56,9 +56,9 @@ class AttestationData(Container): source: Checkpoint target: Checkpoint # Shard data roots - shard_data_roots: List[Hash, MAX_CATCHUP] + shard_data_roots: List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS] # Intermediate state roots - shard_state_roots: List[Hash, MAX_CATCHUP] + shard_state_roots: List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS] # Index index: uint64 ``` @@ -69,7 +69,7 @@ class AttestationData(Container): class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP] + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] signature: BLSSignature ``` @@ -141,7 +141,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ### New state variables ``` - shard_state_roots: Vector[List[Hash, MAX_CATCHUP], MAX_SHARDS] + shard_state_roots: Vector[List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS], MAX_SHARDS] shard_next_slot: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee @@ -177,7 +177,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Correct start slot assert data.slot == state.shard_next_slot[shard] # Correct data root count - assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, MAX_CATCHUP) + max_catchup = ACTIVE_SHARDS * MAX_CATCHUP_RATIO // get_committee_count(state, state.slot) + assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, max_catchup) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Apply @@ -263,7 +264,7 @@ For phase 1, we will use a simple state transition function: Suppose you are a persistent committee member on shard `i` at slot `s`. Suppose `state.shard_next_slots[i] = s-1` ("the happy case"). In this case, you look for a valid proposal that satisfies the checks in the state transition function above, and if you see such a proposal `data` with post-state `post_state`, make an attestation with `shard_data_roots = [hash_tree_root(data)]` and `shard_state_roots = [post_state]`. If you do not find such a proposal, make an attestation using the "default empty proposal", `data = prev_state_root + b'\x00' * 96`. -Now suppose `state.shard_next_slots[i] = s-k` for `k>1`. Then, initialize `data = []`, `states = []`, `state = state.shard_state_roots[i]`. For `slot in (state.shard_next_slot, min(state.shard_next_slot + MAX_CATCHUP, s))`, do: +Now suppose `state.shard_next_slots[i] = s-k` for `k>1`. Then, initialize `data = []`, `states = []`, `state = state.shard_state_roots[i]`. For `slot in (state.shard_next_slot, min(state.shard_next_slot + max_catchup, s))`, do: * Look for all valid proposals for `slot` whose first 32 bytes equal to `state`. If there are none, add a default empty proposal to `data`. If there is one such proposal `p`, add `p` to `data`. If there is more than one, select the one with the largest number of total attestations supporting it or its descendants, and add it to `data`. * Set `state` to the state after processing the proposal just added to `data`; append it to `states` From fe60b4debaf853bea3c7b77e26de5ea4c8fb4223 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 13 Oct 2019 17:13:52 +0900 Subject: [PATCH 005/111] Variable max attestation length --- specs/core/1_new_shards.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 2d199a6409..adc8fb4128 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -211,6 +211,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` +Check the length of attestations using `len(block.attestations) <= 4 * get_committee_count(state, state.slot)`. + ### Light client processing ```python From b792fe696a2ddaa7109f1b1210185c27a2338053 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 13 Oct 2019 17:42:55 +0900 Subject: [PATCH 006/111] formatting --- specs/core/1_new_shards.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index adc8fb4128..19de5773ed 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -140,7 +140,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ### New state variables -``` +```python shard_state_roots: Vector[List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS], MAX_SHARDS] shard_next_slot: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] @@ -150,7 +150,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ### New block data structures -``` +```python light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] light_client_signature: BLSSignature ``` From 9abfc6276a59293529c954eb56d8a3f39b0e14c6 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 14 Oct 2019 18:20:35 +0900 Subject: [PATCH 007/111] Cleaned up state transition and honest committee member --- specs/core/1_new_shards.md | 57 ++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 19de5773ed..32620ef8df 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -20,7 +20,8 @@ - [Light client signature processing)(#light-client-signature-processing) - [Epoch transition](#epoch-transition) - [Fraud proofs](#fraud-proofs) - - [Honest persistent committee member behavior](#honest-persistent-committee-member-behavior) + - [Shard state transition function](#shard-state-transition-function) + - [Honest committee member behavior](#honest-committee-member-behavior) @@ -36,7 +37,6 @@ This document describes the shard transition function (data layer only) and the | - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | -| `SHARD_ROOT_HISTORY_LENGTH` | `2**15` (= 32,768) | | `MAX_CATCHUP_RATIO` | `2**2` (= 4) | | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | @@ -142,7 +142,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ```python shard_state_roots: Vector[List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS], MAX_SHARDS] - shard_next_slot: Vector[Slot, MAX_SHARDS] + shard_next_slots: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee next_light_committee: CompactCommittee @@ -175,7 +175,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Type 1: on-time attestations if data.custody_bits != []: # Correct start slot - assert data.slot == state.shard_next_slot[shard] + assert data.slot == state.shard_next_slots[shard] # Correct data root count max_catchup = ACTIVE_SHARDS * MAX_CATCHUP_RATIO // get_committee_count(state, state.slot) assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, max_catchup) @@ -185,7 +185,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: online_indices = get_online_indices(state) if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: state.shard_state_roots[shard] = data.shard_state_roots - state.shard_next_slot[shard] += len(data.shard_data_roots) + state.shard_next_slots[shard] += len(data.shard_data_roots) # Type 2: delayed attestations else: @@ -254,7 +254,7 @@ if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: TODO. The intent is to have a single universal fraud proof type, which contains (i) an on-time attestation on shard `s` signing a set of `data_roots`, (ii) an index `i` of a particular data root to focus on, (iii) the full contents of the i'th data, (iii) a Merkle proof to the `shard_state_roots` in the parent block the attestation is referencing, and which then verifies that one of the two conditions is false: * `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -* `execute_state_transition(slot, shard, attestation.shard_state_roots[i-1], parent.shard_state_roots, block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s][-1]`) +* `execute_state_transition(shard, slot, attestation.shard_state_roots[i-1], hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s][-1]`) For phase 1, we will use a simple state transition function: @@ -262,13 +262,46 @@ For phase 1, we will use a simple state transition function: * Check that `bls_verify(get_shard_proposer(state, slot, shard), hash_tree_root(data[-96:]), BLSSignature(data[-96:]), BLOCK_SIGNATURE_DOMAIN)` * Output the new state root: `hash_tree_root(prev_state_root, other_prev_state_roots, data)` -## Honest persistent committee member behavior +## Shard state transition function -Suppose you are a persistent committee member on shard `i` at slot `s`. Suppose `state.shard_next_slots[i] = s-1` ("the happy case"). In this case, you look for a valid proposal that satisfies the checks in the state transition function above, and if you see such a proposal `data` with post-state `post_state`, make an attestation with `shard_data_roots = [hash_tree_root(data)]` and `shard_state_roots = [post_state]`. If you do not find such a proposal, make an attestation using the "default empty proposal", `data = prev_state_root + b'\x00' * 96`. +```python +def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: Bytes) -> Hash: + # Beginning of block data is the previous state root + assert block_data[:32] == pre_state + assert block_data[32:64] == int_to_bytes8(slot) + b'\x00' * 24 + # Signature check (nonempty blocks only) + if len(block_data) == 64: + pass + else: + assert len(block_data) >= 160 + assert bls_verify( + pubkey=proposer_pubkey, + message_hash=hash_tree_root(block_data[:-96]), + signature=block_data[-96:], + domain=DOMAIN_SHARD_PROPOSER + ) + # We will add something more substantive in phase 2 + return hash(pre_state + hash_tree_root(block_data)) +``` + +We also provide a method to generate an empty proposal: + +```python +def make_empty_proposal(pre_state: Hash, slot: Slot) -> Bytes[64]: + return pre_state + int_to_bytes8(slot) + b'\x00' * 24 +``` + +## Honest committee member behavior -Now suppose `state.shard_next_slots[i] = s-k` for `k>1`. Then, initialize `data = []`, `states = []`, `state = state.shard_state_roots[i]`. For `slot in (state.shard_next_slot, min(state.shard_next_slot + max_catchup, s))`, do: +Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on. Three seconds into slot `slot`, run the following procedure: -* Look for all valid proposals for `slot` whose first 32 bytes equal to `state`. If there are none, add a default empty proposal to `data`. If there is one such proposal `p`, add `p` to `data`. If there is more than one, select the one with the largest number of total attestations supporting it or its descendants, and add it to `data`. -* Set `state` to the state after processing the proposal just added to `data`; append it to `states` +* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_state_roots[shard][-1]`. +* Let `max_catchup = ACTIVE_SHARDS * MAX_CATCHUP_RATIO // get_committee_count(state, current_slot))` +* For `slot in (state.shard_next_slots[shard], min(state.shard_next_slot + max_catchup, current_slot))`, do the following: + * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. + * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` + * If `len(choices) == 1`, do `proposals.append(choices[0])` + * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. + * Set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. -Make an attestation using `shard_data_roots = data` and `shard_state_roots = states`. +Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. From e5544d055b1a12fb5e9b0d6952c97a3155e9d535 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 14 Oct 2019 23:55:41 +0800 Subject: [PATCH 008/111] Made trace a commitment separate from state roots --- specs/core/1_new_shards.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 32620ef8df..087a8af178 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -91,13 +91,6 @@ def get_online_indices(state: BeaconState) -> Set[ValidatorIndex]: return set([i for i in active_validators if state.online_countdown[i] != 0]) ``` -### `get_shard_state_root` - -```python -def get_shard_state_root(state: BeaconState, shard: Shard) -> Hash: - return state.shard_state_roots[shard][-1] -``` - ### `pack_compact_validator` ```python @@ -141,7 +134,8 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ### New state variables ```python - shard_state_roots: Vector[List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS], MAX_SHARDS] + shard_state_roots: Vector[Hash, MAX_SHARDS] + shard_trace_commitments: Vector[Hash, MAX_SHARDS] shard_next_slots: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee @@ -184,7 +178,13 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Apply online_indices = get_online_indices(state) if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: - state.shard_state_roots[shard] = data.shard_state_roots + # Save trace commitment (used for fraud proofs) + trace = List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS * 2 + 1]([state.shard_state_roots[shard]]) + for data, state in zip(data.shard_data_roots, data.shard_state_roots): + trace.extend([data, state]) + state.shard_trace_commitments[shard] = hash_tree_root(trace) + # Save state root and next slot + state.shard_state_roots[shard] = data.shard_state_roots[-1] state.shard_next_slots[shard] += len(data.shard_data_roots) # Type 2: delayed attestations @@ -256,12 +256,6 @@ TODO. The intent is to have a single universal fraud proof type, which contains * `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` * `execute_state_transition(shard, slot, attestation.shard_state_roots[i-1], hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s][-1]`) -For phase 1, we will use a simple state transition function: - -* Check that `data[:32] == prev_state_root` -* Check that `bls_verify(get_shard_proposer(state, slot, shard), hash_tree_root(data[-96:]), BLSSignature(data[-96:]), BLOCK_SIGNATURE_DOMAIN)` -* Output the new state root: `hash_tree_root(prev_state_root, other_prev_state_roots, data)` - ## Shard state transition function ```python From 7fc2830730284f964a5371daf86c5ab23b52f37f Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 27 Oct 2019 09:01:10 +0800 Subject: [PATCH 009/111] Some updates --- specs/core/1_new_shards.md | 178 ++++++++++++++++++++++++++++--------- 1 file changed, 138 insertions(+), 40 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 087a8af178..54990a43f4 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -41,28 +41,61 @@ This document describes the shard transition function (data layer only) and the | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | +| `SHARD_STATE_ROOT_LENGTH` | `2**7` (= 128) | bytes | +| `MAX_SHARD_BLOCK ## Containers +### Aliases + +| Name | Value | +| - | - | +| `SHARD_STATE_ROOT` | `BytesN[SHARD_STATE_ROOT_LENGTH]` | + + ### `AttestationData` ```python class AttestationData(Container): - # Slot slot: Slot + index: CommitteeIndex # LMD GHOST vote beacon_block_root: Hash # FFG vote source: Checkpoint target: Checkpoint + # Shard data + shard_data: AttestationShardData +``` + +### `AttestationShardData` + +```python +class AttestationShardData(Container): + # Shard block lengths + shard_block_lengths: List[uint8, MAX_CATCHUP_RATIO * MAX_SHARDS] # Shard data roots shard_data_roots: List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS] # Intermediate state roots - shard_state_roots: List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS] - # Index - index: uint64 + shard_state_roots: List[SHARD_STATE_ROOT, MAX_CATCHUP_RATIO * MAX_SHARDS] +``` + +### `ReducedAttestationData` + +```python +class ReducedAttestationData(Container): + slot: Slot + index: CommitteeIndex + # LMD GHOST vote + beacon_block_root: Hash + # FFG vote + source: Checkpoint + target: Checkpoint + # Shard data root + shard_data_root: Hash ``` + ### `Attestation` ```python @@ -73,6 +106,26 @@ class Attestation(Container): signature: BLSSignature ``` +### `ReducedAttestation` + +```python +class ReducedAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: ReducedAttestationData + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] + signature: BLSSignature +``` + +### `IndexedAttestation` + +```python +class IndexedAttestation(Container): + participants: List[ValidatorIndex, MAX_COMMITTEE_SIZE] + data: ReducedAttestationData + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] + signature: BLSSignature +``` + ### `CompactCommittee` ```python @@ -81,6 +134,15 @@ class CompactCommittee(Container): compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] ``` +### `AttestationCustodyBitWrapper` + +``` +class AttestationCustodyBitWrapper(Container): + attestation_root: Hash + index: uint64 + bit: bool +``` + ## Helpers ### `get_online_validators` @@ -103,16 +165,6 @@ def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int return (index << 16) + (slashed << 15) + balance_in_increments ``` -### `unpack_compact_validator` - -```python -def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: - """ - Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT - """ - return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1) -``` - ### `committee_to_compact_committee` ```python @@ -129,6 +181,52 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` +### `get_light_client_committee` + +```python +def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + assert epoch % LIGHT_CLIENT_COMMITTEE_PERIOD == 0 + active_validator_indices = get_active_validator_indices(beacon_state, epoch) + seed = get_seed(beacon_state, epoch, DOMAIN_SHARD_LIGHT_CLIENT) + return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS)[:TARGET_COMMITTEE_SIZE] +``` + +### `get_indexed_attestation` + +```python +def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + return IndexedAttestation(attesting_indices, data, custody_bits, signature) +``` + +### `is_valid_indexed_attestation` + +``python +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Check if ``indexed_attestation`` has valid indices and signature. + """ + + # Verify indices are sorted + if indexed_attestation.participants != sorted(indexed_attestation.participants): + return False + + # Verify aggregate signature + all_pubkeys = [] + all_message_hashes = [] + for participant, custody_bits in zip(participants, indexed_attestation.custody_bits): + for i, bit in enumerate(custody_bits): + all_pubkeys.append(state.validators[participant].pubkey) + all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, bit)) + + return bls_verify_multiple( + pubkeys=all_pubkeys, + message_hashes=all_message_hashes, + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch), + ) +``` + ## Beacon Chain Changes ### New state variables @@ -216,37 +314,37 @@ Check the length of attestations using `len(block.attestations) <= 4 * get_commi ### Light client processing ```python -signer_validators = [] -signer_keys = [] -for i, bit in enumerate(block.light_client_signature_bitfield): - if bit: - signer_keys.append(state.current_light_committee.pubkeys[i]) - index, _, _ = unpack_compact_validator(state.current_light_committee.compact_validators[i]) - signer_validators.append(index) - -assert bls_verify( - pubkey=bls_aggregate_pubkeys(signer_keys), - message_hash=get_block_root_at_slot(state, state.slot - 1), - signature=block.light_client_signature, - domain=DOMAIN_LIGHT_CLIENT -) +def verify_light_client_signatures(state: BeaconState, block: BeaconBlock): + period_start = get_current_epoch(state) - get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD + committee = get_light_client_committee(state, period_start - min(period_start, LIGHT_CLIENT_COMMITTEE_PERIOD)) + signer_validators = [] + signer_keys = [] + for i, bit in enumerate(block.light_client_signature_bitfield): + if bit: + signer_keys.append(state.validators[committee[i]].pubkey) + signer_validators.append(committee[i]) + + assert bls_verify( + pubkey=bls_aggregate_pubkeys(signer_keys), + message_hash=get_block_root_at_slot(state, state.slot - 1), + signature=block.light_client_signature, + domain=DOMAIN_LIGHT_CLIENT + ) ``` ### Epoch transition ```python -# Slowly remove validators from the "online" set if they do not show up -for index in range(len(state.validators)): - if state.online_countdown[index] != 0: - state.online_countdown[index] = state.online_countdown[index] - 1 - -# Update light client committees -if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: - state.current_light_committee = state.next_light_committee - seed = get_seed(state, get_current_epoch(state), DOMAIN_LIGHT_CLIENT) - active_indices = get_active_validator_indices(state, get_current_epoch(state)) - committee = [active_indices[compute_shuffled_index(ValidatorIndex(i), len(active_indices), seed)] for i in range(LIGHT_CLIENT_COMMITTEE_SIZE)] - state.next_light_committee = committee_to_compact_committee(state, committee) +def phase_1_epoch_transition(state): + # Slowly remove validators from the "online" set if they do not show up + for index in range(len(state.validators)): + if state.online_countdown[index] != 0: + state.online_countdown[index] = state.online_countdown[index] - 1 + + # Update light client committees + if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: + state.current_light_committee = state.next_light_committee + state.next_light_committee = committee_to_compact_committee(state, get_light_client_committee(state, get_current_epoch(state))) ``` ### Fraud proofs From bb2835ee1bfc48808f3f15150f5187dbbcffc9b9 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 28 Oct 2019 02:01:22 +0800 Subject: [PATCH 010/111] Shard slots to 128 bytes --- specs/core/1_new_shards.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 54990a43f4..0701bfa9fc 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -42,7 +42,7 @@ This document describes the shard transition function (data layer only) and the | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | | `SHARD_STATE_ROOT_LENGTH` | `2**7` (= 128) | bytes | -| `MAX_SHARD_BLOCK +| `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | ## Containers @@ -75,7 +75,7 @@ class AttestationShardData(Container): # Shard block lengths shard_block_lengths: List[uint8, MAX_CATCHUP_RATIO * MAX_SHARDS] # Shard data roots - shard_data_roots: List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS] + shard_data_roots: List[Hash, List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_CATCHUP_RATIO * MAX_SHARDS] # Intermediate state roots shard_state_roots: List[SHARD_STATE_ROOT, MAX_CATCHUP_RATIO * MAX_SHARDS] ``` @@ -232,7 +232,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ### New state variables ```python - shard_state_roots: Vector[Hash, MAX_SHARDS] + shard_state_roots: Vector[SHARD_STATE_ROOT, MAX_SHARDS] shard_trace_commitments: Vector[Hash, MAX_SHARDS] shard_next_slots: Vector[Slot, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] From 539c6819213479661d0d38a27b90bdda4c77b36e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 29 Oct 2019 10:43:13 -0700 Subject: [PATCH 011/111] Updates to add new proposals --- specs/core/1_new_shards.md | 165 ++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 68 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 0701bfa9fc..cb2348d657 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -37,21 +37,27 @@ This document describes the shard transition function (data layer only) and the | - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | -| `MAX_CATCHUP_RATIO` | `2**2` (= 4) | | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | -| `SHARD_STATE_ROOT_LENGTH` | `2**7` (= 128) | bytes | +| `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | | `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | +| `BLOCK_SIZE_TARGET` | `3 * 2**16` (= 196,608) | | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | +| `MAX_SHARD_GASPRICE` | `2**14` (= 16,384) | Gwei | | +| `SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | ## Containers -### Aliases - -| Name | Value | -| - | - | -| `SHARD_STATE_ROOT` | `BytesN[SHARD_STATE_ROOT_LENGTH]` | +### `ShardState` +```python +class ShardState(Container): + slot: Slot + gasprice: Gwei + root: Hash +``` ### `AttestationData` @@ -64,55 +70,31 @@ class AttestationData(Container): # FFG vote source: Checkpoint target: Checkpoint - # Shard data - shard_data: AttestationShardData + # Shard transition hash + shard_transition_hash: Hash ``` -### `AttestationShardData` +### `ShardTransition` ```python class AttestationShardData(Container): + # Starting from slot + start_slot: Slot # Shard block lengths - shard_block_lengths: List[uint8, MAX_CATCHUP_RATIO * MAX_SHARDS] + shard_block_lengths: List[uint8, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots - shard_data_roots: List[Hash, List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_CATCHUP_RATIO * MAX_SHARDS] + shard_data_roots: List[Hash, List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate state roots - shard_state_roots: List[SHARD_STATE_ROOT, MAX_CATCHUP_RATIO * MAX_SHARDS] -``` - -### `ReducedAttestationData` - -```python -class ReducedAttestationData(Container): - slot: Slot - index: CommitteeIndex - # LMD GHOST vote - beacon_block_root: Hash - # FFG vote - source: Checkpoint - target: Checkpoint - # Shard data root - shard_data_root: Hash + shard_state_roots: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] ``` - ### `Attestation` ```python class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] - signature: BLSSignature -``` - -### `ReducedAttestation` - -```python -class ReducedAttestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - data: ReducedAttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -121,8 +103,8 @@ class ReducedAttestation(Container): ```python class IndexedAttestation(Container): participants: List[ValidatorIndex, MAX_COMMITTEE_SIZE] - data: ReducedAttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_CATCHUP_RATIO * MAX_SHARDS] + data: AttestationData + custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -199,6 +181,21 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) return IndexedAttestation(attesting_indices, data, custody_bits, signature) ``` +### `update_gasprice` + +```python +def update_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: + if length > BLOCK_SIZE_TARGET: + delta = prev_gasprice * (length - BLOCK_SIZE_TARGET) // BLOCK_SIZE_TARGET // SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT + return min(prev_gasprice + delta, MAX_SHARD_GASPRICE) + else: + delta = prev_gasprice * (BLOCK_SIZE_TARGET - length) // BLOCK_SIZE_TARGET // SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT + if delta > prev_gasprice - SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT: + return SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT + else: + return prev_gasprice - delta +``` + ### `is_valid_indexed_attestation` ``python @@ -217,6 +214,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe for participant, custody_bits in zip(participants, indexed_attestation.custody_bits): for i, bit in enumerate(custody_bits): all_pubkeys.append(state.validators[participant].pubkey) + # Note: only 2N distinct message hashes all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, bit)) return bls_verify_multiple( @@ -232,9 +230,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ### New state variables ```python - shard_state_roots: Vector[SHARD_STATE_ROOT, MAX_SHARDS] - shard_trace_commitments: Vector[Hash, MAX_SHARDS] - shard_next_slots: Vector[Slot, MAX_SHARDS] + shard_transitions: Vector[ShardTransition, MAX_SHARDS] + shard_states: Vector[ShardState, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee next_light_committee: CompactCommittee @@ -254,52 +251,76 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data assert data.index < ACTIVE_SHARDS shard = (data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS + proposer_index=get_beacon_proposer_index(state) # Signature check committee = get_crosslink_committee(state, get_current_epoch(state), shard) for bits in attestation.custody_bits + [attestation.aggregation_bits]: - assert bits == len(committee) + assert len(bits) == len(committee) # Check signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Get attesting indices attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + + # Prepare pending attestation object + pending_attestation = PendingAttestation( + slot=data.slot, + shard=shard, + aggregation_bits=attestation.aggregation_bits, + inclusion_delay=state.slot - data.slot, + crosslink_success=False, + proposer_index=proposer_index + ) # Type 1: on-time attestations if data.custody_bits != []: - # Correct start slot - assert data.slot == state.shard_next_slots[shard] + # Correct slot + assert data.slot == state.slot + # Slot the attestation starts counting from + start_slot = state.shard_next_slots[shard] # Correct data root count - max_catchup = ACTIVE_SHARDS * MAX_CATCHUP_RATIO // get_committee_count(state, state.slot) - assert len(data.shard_data_roots) == len(attestation.custody_bits) == len(data.shard_state_roots) == min(state.slot - data.slot, max_catchup) + offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] + assert len(attestation.custody_bits) == len(offset_slots) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Apply online_indices = get_online_indices(state) if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: - # Save trace commitment (used for fraud proofs) - trace = List[Hash, MAX_CATCHUP_RATIO * MAX_SHARDS * 2 + 1]([state.shard_state_roots[shard]]) - for data, state in zip(data.shard_data_roots, data.shard_state_roots): - trace.extend([data, state]) - state.shard_trace_commitments[shard] = hash_tree_root(trace) - # Save state root and next slot - state.shard_state_roots[shard] = data.shard_state_roots[-1] - state.shard_next_slots[shard] += len(data.shard_data_roots) - + # Check correct formatting of shard transition data + transition = block.shard_transitions[shard] + assert data.shard_transition_hash == hash_tree_root(transition) + assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) + assert transition.start_slot == start_slot + + # Verify correct calculation of gas prices and slots and chunk roots + prev_gasprice = state.shard_states[shard].gasprice + for i in range(len(offset_slots)): + assert transition.shard_states[i].gasprice == update_gasprice(prev_gasprice, transition.shard_block_lengths[i]) + assert transition.shard_states[i].slot == offset_slots[i] + assetrt len(transition.shard_data_roots[i]) == transition.shard_block_lengths[i] // SHARD_BLOCK_CHUNK_SIZE + prev_gasprice = transition.shard_states[i].gasprice + + # Save updated state + state.shard_states[shard] = data.shard_states[-1] + state.shard_states[shard].slot = state.slot - 1 + + # Save success (for end-of-epoch rewarding) + pending_attestation.crosslink_success = True + + # Reward and cost proposer + estimated_attester_reward = sum([get_base_reward(state, attester) for attester in attesting_indices]) + increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) + for state, length in zip(transition.shard_states, transition.shard_block_lengths): + decrease_balance(state, proposer, state.gasprice * length) + # Type 2: delayed attestations else: assert slot_to_epoch(data.slot) in (get_current_epoch(state), get_previous_epoch(state)) - assert len(data.shard_data_roots) == len(data.intermediate_state_roots) == 0 + assert len(attestation.custody_bits) == 0 for index in attesting_indices: online_countdown[index] = ONLINE_PERIOD - pending_attestation = PendingAttestation( - slot=data.slot, - shard=shard, - aggregation_bits=attestation.aggregation_bits, - inclusion_delay=state.slot - attestation_slot, - proposer_index=get_beacon_proposer_index(state), - ) if data.target.epoch == get_current_epoch(state): assert data.source == state.current_justified_checkpoint @@ -309,7 +330,15 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` -Check the length of attestations using `len(block.attestations) <= 4 * get_committee_count(state, state.slot)`. +### Misc block post-processing + +```python +def misc_block_post_process(state: BeaconState, block: BeaconBlock): + # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it + for shard in range(MAX_SHARDS): + if state.shard_states[shard].slot != state.slot - 1: + assert block.shard_transition[shard] == ShardTransition() +``` ### Light client processing From 0ff5985c010a442cd3cce7414bd4ad0fbd1f7686 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 28 Oct 2019 17:18:27 +0800 Subject: [PATCH 012/111] Fix markdown and ToC --- specs/core/1_new_shards.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index cb2348d657..7721bcf5a9 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -6,18 +6,33 @@ -- [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains) +- [Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data](#ethereum-20-phase-1----crosslinks-and-shard-data) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Configuration](#configuration) - [Misc](#misc) - [Containers](#containers) + - [Aliases](#aliases) + - [`AttestationData`](#attestationdata) + - [`AttestationShardData`](#attestationsharddata) + - [`ReducedAttestationData`](#reducedattestationdata) + - [`Attestation`](#attestation) + - [`ReducedAttestation`](#reducedattestation) + - [`IndexedAttestation`](#indexedattestation) + - [`CompactCommittee`](#compactcommittee) + - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - [Helpers](#helpers) + - [`get_online_validators`](#get_online_validators) + - [`pack_compact_validator`](#pack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`get_light_client_committee`](#get_light_client_committee) + - [`get_indexed_attestation`](#get_indexed_attestation) + - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - [Beacon Chain Changes](#beacon-chain-changes) - [New state variables](#new-state-variables) - [New block data structures](#new-block-data-structures) - [Attestation processing](#attestation-processing) - - [Light client signature processing)(#light-client-signature-processing) + - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) - [Fraud proofs](#fraud-proofs) - [Shard state transition function](#shard-state-transition-function) @@ -118,7 +133,7 @@ class CompactCommittee(Container): ### `AttestationCustodyBitWrapper` -``` +```python class AttestationCustodyBitWrapper(Container): attestation_root: Hash index: uint64 @@ -198,7 +213,7 @@ def update_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: ### `is_valid_indexed_attestation` -``python +```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ Check if ``indexed_attestation`` has valid indices and signature. From 08fc024fbd6e553116dc4156d9377e38dfa7d96c Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 29 Oct 2019 10:56:58 -0700 Subject: [PATCH 013/111] Changed fraud proof details --- specs/core/1_new_shards.md | 47 +++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 7721bcf5a9..13cd4fe8a6 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -60,6 +60,7 @@ This document describes the shard transition function (data layer only) and the | `BLOCK_SIZE_TARGET` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | +| `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_SHARD_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | @@ -72,6 +73,7 @@ class ShardState(Container): slot: Slot gasprice: Gwei root: Hash + latest_block_hash: Hash ``` ### `AttestationData` @@ -312,7 +314,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: for i in range(len(offset_slots)): assert transition.shard_states[i].gasprice == update_gasprice(prev_gasprice, transition.shard_block_lengths[i]) assert transition.shard_states[i].slot == offset_slots[i] - assetrt len(transition.shard_data_roots[i]) == transition.shard_block_lengths[i] // SHARD_BLOCK_CHUNK_SIZE + assert len(transition.shard_data_roots[i]) == transition.shard_block_lengths[i] // SHARD_BLOCK_CHUNK_SIZE + filled_roots = transition.shard_data_roots + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(transition.shard_data_roots)) + assert transition.shard_states[i].latest_block_hash == hash_tree_root(filled_roots) prev_gasprice = transition.shard_states[i].gasprice # Save updated state @@ -401,39 +405,30 @@ TODO. The intent is to have a single universal fraud proof type, which contains ## Shard state transition function ```python -def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: Bytes) -> Hash: - # Beginning of block data is the previous state root - assert block_data[:32] == pre_state +def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: + # Beginning of block data is the previous block hash + assert block_data[:32] == pre_state.latest_block_hash assert block_data[32:64] == int_to_bytes8(slot) + b'\x00' * 24 - # Signature check (nonempty blocks only) - if len(block_data) == 64: - pass - else: - assert len(block_data) >= 160 - assert bls_verify( - pubkey=proposer_pubkey, - message_hash=hash_tree_root(block_data[:-96]), - signature=block_data[-96:], - domain=DOMAIN_SHARD_PROPOSER - ) + # Signature check + assert len(block_data) >= 160 + assert bls_verify( + pubkey=proposer_pubkey, + message_hash=hash_tree_root(block_data[:-96]), + signature=block_data[-96:], + domain=DOMAIN_SHARD_PROPOSER + ) # We will add something more substantive in phase 2 - return hash(pre_state + hash_tree_root(block_data)) -``` - -We also provide a method to generate an empty proposal: - -```python -def make_empty_proposal(pre_state: Hash, slot: Slot) -> Bytes[64]: - return pre_state + int_to_bytes8(slot) + b'\x00' * 24 + length = len(block.data.rstrip(b'\x00')) + return ShardState(slot=slot, root=hash(pre_state + hash_tree_root(block_data)), gasprice=update_gasprice(pre_state, length), latest_block_hash = hash(block_data)) ``` ## Honest committee member behavior Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on. Three seconds into slot `slot`, run the following procedure: -* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_state_roots[shard][-1]`. -* Let `max_catchup = ACTIVE_SHARDS * MAX_CATCHUP_RATIO // get_committee_count(state, current_slot))` -* For `slot in (state.shard_next_slots[shard], min(state.shard_next_slot + max_catchup, current_slot))`, do the following: +* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. +* Let `offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot]` +* For `slot in offset_slots`, do the following: * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` * If `len(choices) == 1`, do `proposals.append(choices[0])` From bf0a2b7df983fdd0da92eed5f8f699411931b61e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 29 Oct 2019 11:12:55 -0700 Subject: [PATCH 014/111] Renaming --- specs/core/1_new_shards.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 13cd4fe8a6..7f7a4dcddf 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -61,8 +61,8 @@ This document describes the shard transition function (data layer only) and the | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | -| `MAX_SHARD_GASPRICE` | `2**14` (= 16,384) | Gwei | | -| `SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | +| `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | +| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | ## Containers @@ -203,12 +203,12 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ```python def update_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: if length > BLOCK_SIZE_TARGET: - delta = prev_gasprice * (length - BLOCK_SIZE_TARGET) // BLOCK_SIZE_TARGET // SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT - return min(prev_gasprice + delta, MAX_SHARD_GASPRICE) + delta = prev_gasprice * (length - BLOCK_SIZE_TARGET) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT + return min(prev_gasprice + delta, MAX_GASPRICE) else: - delta = prev_gasprice * (BLOCK_SIZE_TARGET - length) // BLOCK_SIZE_TARGET // SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT - if delta > prev_gasprice - SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT: - return SHARD_GASPRICE_ADJUSTMENT_COEFFICIENT + delta = prev_gasprice * (BLOCK_SIZE_TARGET - length) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT + if delta > prev_gasprice - GASPRICE_ADJUSTMENT_COEFFICIENT: + return GASPRICE_ADJUSTMENT_COEFFICIENT else: return prev_gasprice - delta ``` From bf886d698bc37935c410054ba4216397e72be79d Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 29 Oct 2019 11:33:29 -0700 Subject: [PATCH 015/111] Small edits --- specs/core/1_new_shards.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 7f7a4dcddf..0d0abfbd7b 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -94,7 +94,7 @@ class AttestationData(Container): ### `ShardTransition` ```python -class AttestationShardData(Container): +class ShardTransition(Container): # Starting from slot start_slot: Slot # Shard block lengths @@ -247,7 +247,6 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ### New state variables ```python - shard_transitions: Vector[ShardTransition, MAX_SHARDS] shard_states: Vector[ShardState, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee @@ -257,6 +256,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ### New block data structures ```python + shard_transitions: Vector[ShardTransition, MAX_SHARDS] light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] light_client_signature: BLSSignature ``` @@ -312,21 +312,23 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Verify correct calculation of gas prices and slots and chunk roots prev_gasprice = state.shard_states[shard].gasprice for i in range(len(offset_slots)): - assert transition.shard_states[i].gasprice == update_gasprice(prev_gasprice, transition.shard_block_lengths[i]) - assert transition.shard_states[i].slot == offset_slots[i] - assert len(transition.shard_data_roots[i]) == transition.shard_block_lengths[i] // SHARD_BLOCK_CHUNK_SIZE - filled_roots = transition.shard_data_roots + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(transition.shard_data_roots)) - assert transition.shard_states[i].latest_block_hash == hash_tree_root(filled_roots) - prev_gasprice = transition.shard_states[i].gasprice + shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] + block_length = transition.shard + assert shard_state.gasprice == update_gasprice(prev_gasprice, block_length) + assert shard_state.slot == offset_slots[i] + assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE + filled_roots = chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) + assert shard_state.latest_block_hash == hash_tree_root(filled_roots) + prev_gasprice = shard_state.gasprice # Save updated state - state.shard_states[shard] = data.shard_states[-1] + state.shard_states[shard] = transition.shard_states[-1] state.shard_states[shard].slot = state.slot - 1 # Save success (for end-of-epoch rewarding) pending_attestation.crosslink_success = True - # Reward and cost proposer + # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in attesting_indices]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) for state, length in zip(transition.shard_states, transition.shard_block_lengths): @@ -335,6 +337,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Type 2: delayed attestations else: assert slot_to_epoch(data.slot) in (get_current_epoch(state), get_previous_epoch(state)) + assert data.shard_transition_hash == Hash() assert len(attestation.custody_bits) == 0 for index in attesting_indices: @@ -365,12 +368,11 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock): def verify_light_client_signatures(state: BeaconState, block: BeaconBlock): period_start = get_current_epoch(state) - get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD committee = get_light_client_committee(state, period_start - min(period_start, LIGHT_CLIENT_COMMITTEE_PERIOD)) - signer_validators = [] signer_keys = [] for i, bit in enumerate(block.light_client_signature_bitfield): if bit: signer_keys.append(state.validators[committee[i]].pubkey) - signer_validators.append(committee[i]) + increase_balance(state, committee[i], get_base_reward(state, committee[i])) assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), From b36820a26c145708bf43cd014e079ba8c6dc58a5 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Fri, 1 Nov 2019 07:58:13 -0700 Subject: [PATCH 016/111] Changes to make Danny happy --- specs/core/1_new_shards.md | 77 +++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 0d0abfbd7b..4089c5478d 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -29,8 +29,8 @@ - [`get_indexed_attestation`](#get_indexed_attestation) - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - [Beacon Chain Changes](#beacon-chain-changes) - - [New state variables](#new-state-variables) - - [New block data structures](#new-block-data-structures) + - [New beacon state fields](#new-beacon-state-fields) + - [New beacon block data fields](#new-beacon-block-data-fields) - [Attestation processing](#attestation-processing) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) @@ -63,6 +63,7 @@ This document describes the shard transition function (data layer only) and the | `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | +| `DOMAIN_SHARD_LIGHT_CLIENT` | `192` | | ## Containers @@ -87,8 +88,8 @@ class AttestationData(Container): # FFG vote source: Checkpoint target: Checkpoint - # Shard transition hash - shard_transition_hash: Hash + # Shard transition root + shard_transition_root: Hash ``` ### `ShardTransition` @@ -98,9 +99,9 @@ class ShardTransition(Container): # Starting from slot start_slot: Slot # Shard block lengths - shard_block_lengths: List[uint8, MAX_SHARD_BLOCKS_PER_ATTESTATION] + shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots - shard_data_roots: List[Hash, List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] + shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate state roots shard_state_roots: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] ``` @@ -119,7 +120,7 @@ class Attestation(Container): ```python class IndexedAttestation(Container): - participants: List[ValidatorIndex, MAX_COMMITTEE_SIZE] + participants: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature @@ -138,10 +139,21 @@ class CompactCommittee(Container): ```python class AttestationCustodyBitWrapper(Container): attestation_root: Hash - index: uint64 + block_index: uint64 bit: bool ``` +### `PendingAttestation` + +```python +class PendingAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + inclusion_delay: Slot + proposer_index: ValidatorIndex + crosslink_success: bool +``` + ## Helpers ### `get_online_validators` @@ -184,9 +196,11 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: - assert epoch % LIGHT_CLIENT_COMMITTEE_PERIOD == 0 - active_validator_indices = get_active_validator_indices(beacon_state, epoch) - seed = get_seed(beacon_state, epoch, DOMAIN_SHARD_LIGHT_CLIENT) + source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD + if source_epoch > 0: + source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD + active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) + seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_LIGHT_CLIENT) return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS)[:TARGET_COMMITTEE_SIZE] ``` @@ -195,7 +209,7 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - return IndexedAttestation(attesting_indices, data, custody_bits, signature) + return IndexedAttestation(attesting_indices, attestation.data, attestation.custody_bits, attestation.signature) ``` ### `update_gasprice` @@ -228,8 +242,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify aggregate signature all_pubkeys = [] all_message_hashes = [] - for participant, custody_bits in zip(participants, indexed_attestation.custody_bits): - for i, bit in enumerate(custody_bits): + for i, custody_bits in enumerate(indexed_attestation.custody_bits): + for participant, bit in zip(participants, custody_bits): all_pubkeys.append(state.validators[participant].pubkey) # Note: only 2N distinct message hashes all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, bit)) @@ -244,7 +258,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ## Beacon Chain Changes -### New state variables +### New beacon state fields ```python shard_states: Vector[ShardState, MAX_SHARDS] @@ -253,7 +267,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe next_light_committee: CompactCommittee ``` -### New block data structures +### New beacon block data fields ```python shard_transitions: Vector[ShardTransition, MAX_SHARDS] @@ -268,10 +282,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data assert data.index < ACTIVE_SHARDS shard = (data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS - proposer_index=get_beacon_proposer_index(state) + proposer_index = get_beacon_proposer_index(state) # Signature check - committee = get_crosslink_committee(state, get_current_epoch(state), shard) + committee = get_beacon_committee(state, get_current_epoch(state), shard) for bits in attestation.custody_bits + [attestation.aggregation_bits]: assert len(bits) == len(committee) # Check signature @@ -281,8 +295,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Prepare pending attestation object pending_attestation = PendingAttestation( - slot=data.slot, - shard=shard, aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - data.slot, crosslink_success=False, @@ -305,7 +317,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: # Check correct formatting of shard transition data transition = block.shard_transitions[shard] - assert data.shard_transition_hash == hash_tree_root(transition) + assert data.shard_transition_root == hash_tree_root(transition) assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) assert transition.start_slot == start_slot @@ -331,13 +343,13 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in attesting_indices]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) - for state, length in zip(transition.shard_states, transition.shard_block_lengths): - decrease_balance(state, proposer, state.gasprice * length) + for shard_state, slot, length in zip(transition.shard_states, offset_slots, transition.shard_block_lengths): + decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) # Type 2: delayed attestations else: - assert slot_to_epoch(data.slot) in (get_current_epoch(state), get_previous_epoch(state)) - assert data.shard_transition_hash == Hash() + assert state.slot - slot_to_epoch(data.slot) < EPOCH_LENGTH + assert data.shard_transition_root == Hash() assert len(attestation.custody_bits) == 0 for index in attesting_indices: @@ -365,14 +377,18 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock): ### Light client processing ```python -def verify_light_client_signatures(state: BeaconState, block: BeaconBlock): - period_start = get_current_epoch(state) - get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD - committee = get_light_client_committee(state, period_start - min(period_start, LIGHT_CLIENT_COMMITTEE_PERIOD)) +def process_light_client_signatures(state: BeaconState, block: BeaconBlock): + committee = get_light_client_committee(state, get_current_epoch(state)) + assert len(block.light_client_signature_bitfield) == len(committee) + tot_reward = 0 signer_keys = [] for i, bit in enumerate(block.light_client_signature_bitfield): if bit: signer_keys.append(state.validators[committee[i]].pubkey) increase_balance(state, committee[i], get_base_reward(state, committee[i])) + tot_reward += get_base_reward(state, committee[i]) + + increase_balance(state, get_beacon_proposer_index(state), tot_reward // PROPOSER_REWARD_COEFFICIENT) assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), @@ -394,7 +410,8 @@ def phase_1_epoch_transition(state): # Update light client committees if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: state.current_light_committee = state.next_light_committee - state.next_light_committee = committee_to_compact_committee(state, get_light_client_committee(state, get_current_epoch(state))) + new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) + state.next_light_committee = committee_to_compact_committee(state, new_committee) ``` ### Fraud proofs @@ -420,7 +437,7 @@ def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_b domain=DOMAIN_SHARD_PROPOSER ) # We will add something more substantive in phase 2 - length = len(block.data.rstrip(b'\x00')) + length = len(block_data.rstrip(b'\x00')) return ShardState(slot=slot, root=hash(pre_state + hash_tree_root(block_data)), gasprice=update_gasprice(pre_state, length), latest_block_hash = hash(block_data)) ``` From 6b1bc1e02f0a1be575c911e3725f384349d96891 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 3 Nov 2019 08:17:46 -0800 Subject: [PATCH 017/111] Reformed attestations --- specs/core/1_new_shards.md | 148 +++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 57 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 4089c5478d..cce39ab596 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -256,6 +256,13 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ) ``` +### `get_attestation_shard` + +```python +def get_shard(state: BeaconState, attestation: Attestation): + return (attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS +``` + ## Beacon Chain Changes ### New beacon state fields @@ -275,32 +282,22 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe light_client_signature: BLSSignature ``` -### Attestation processing +## Attestation processing + +### `validate_attestation` ```python -def process_attestation(state: BeaconState, attestation: Attestation) -> None: +def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: data = attestation.data assert data.index < ACTIVE_SHARDS - shard = (data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS + shard = get_shard(state, attestation) proposer_index = get_beacon_proposer_index(state) # Signature check committee = get_beacon_committee(state, get_current_epoch(state), shard) for bits in attestation.custody_bits + [attestation.aggregation_bits]: assert len(bits) == len(committee) - # Check signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # Get attesting indices - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - - # Prepare pending attestation object - pending_attestation = PendingAttestation( - aggregation_bits=attestation.aggregation_bits, - inclusion_delay=state.slot - data.slot, - crosslink_success=False, - proposer_index=proposer_index - ) - # Type 1: on-time attestations if data.custody_bits != []: # Correct slot @@ -312,56 +309,93 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert len(attestation.custody_bits) == len(offset_slots) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) - # Apply - online_indices = get_online_indices(state) - if get_total_balance(state, online_indices.intersection(attesting_indices)) * 3 >= get_total_balance(state, online_indices) * 2: - # Check correct formatting of shard transition data - transition = block.shard_transitions[shard] - assert data.shard_transition_root == hash_tree_root(transition) - assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) - assert transition.start_slot == start_slot - - # Verify correct calculation of gas prices and slots and chunk roots - prev_gasprice = state.shard_states[shard].gasprice - for i in range(len(offset_slots)): - shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] - block_length = transition.shard - assert shard_state.gasprice == update_gasprice(prev_gasprice, block_length) - assert shard_state.slot == offset_slots[i] - assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE - filled_roots = chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) - assert shard_state.latest_block_hash == hash_tree_root(filled_roots) - prev_gasprice = shard_state.gasprice - - # Save updated state - state.shard_states[shard] = transition.shard_states[-1] - state.shard_states[shard].slot = state.slot - 1 - - # Save success (for end-of-epoch rewarding) - pending_attestation.crosslink_success = True - - # Apply proposer reward and cost - estimated_attester_reward = sum([get_base_reward(state, attester) for attester in attesting_indices]) - increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) - for shard_state, slot, length in zip(transition.shard_states, offset_slots, transition.shard_block_lengths): - decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) - # Type 2: delayed attestations else: assert state.slot - slot_to_epoch(data.slot) < EPOCH_LENGTH assert data.shard_transition_root == Hash() assert len(attestation.custody_bits) == 0 +``` - for index in attesting_indices: - online_countdown[index] = ONLINE_PERIOD +### `apply_shard_transition` + +```python +def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: + # Slot the attestation starts counting from + start_slot = state.shard_next_slots[shard] + # Correct data root count + offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] + assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) + assert transition.start_slot == start_slot + + # Verify correct calculation of gas prices and slots and chunk roots + prev_gasprice = state.shard_states[shard].gasprice + for i in range(len(offset_slots)): + shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] + block_length = transition.shard + assert shard_state.gasprice == update_gasprice(prev_gasprice, block_length) + assert shard_state.slot == offset_slots[i] + assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE + filled_roots = chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) + assert shard_state.latest_block_hash == hash_tree_root(filled_roots) + prev_gasprice = shard_state.gasprice + + # Save updated state + state.shard_states[shard] = transition.shard_states[-1] + state.shard_states[shard].slot = state.slot - 1 +``` +### `process_attestations` - if data.target.epoch == get_current_epoch(state): - assert data.source == state.current_justified_checkpoint - state.current_epoch_attestations.append(pending_attestation) - else: - assert data.source == state.previous_justified_checkpoint - state.previous_epoch_attestations.append(pending_attestation) +```python +def process_attestations(state: BeaconState, block: BeaconBlock, attestations: Sequence[Attestation]) -> None: + pending_attestations = [] + # Basic validation + for attestation in attestations: + assert validate_attestation(state, attestation) + # Process crosslinks + online_indices = get_online_indices(state) + winners = set() + for shard in range(ACTIVE_SHARDS): + # All attestations in the block for this shard + this_shard_attestations = [attestation for attestation in attestations if get_shard(state, attestation) == shard and attestation.data.slot == state.slot] + # The committee for this shard + this_shard_committee = get_beacon_committee(state, get_current_epoch(state), shard) + # Loop over all shard transition roots + for shard_transition_root in sorted(set([attestation.data.shard_transition_root for attestation in this_shard_attestations])): + all_participants = set() + participating_attestations = [] + for attestation in this_shard_attestations: + participating_attestations.append(attestation) + if attestation.data.shard_transition_root == shard_transition_root: + all_participants = all_participants.union(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) + if ( + get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= + get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 + ): + assert shard_transition_root == hash_tree_root(block.shard_transition) + process_crosslink(state, shard, block.shard_transition) + # Apply proposer reward and cost + estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) + increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) + for shard_state, slot, length in zip(transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): + decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) + winners.add((shard, shard_transition_root)) + for index in all_participants: + online_countdown[index] = ONLINE_PERIOD + for attestation in attestations: + pending_attestation = PendingAttestation( + aggregation_bits=attestation.aggregation_bits, + data=attestation.data, + inclusion_delay=state.slot - data.slot, + crosslink_success=(attestation.shard, attestation.shard_transition_root) in winners and attestation.data.slot == state.slot, + proposer_index=proposer_index + ) + if attestation.data.target.epoch == get_current_epoch(state): + assert attestation.data.source == state.current_justified_checkpoint + state.current_epoch_attestations.append(pending_attestation) + else: + assert attestation.data.source == state.previous_justified_checkpoint + state.previous_epoch_attestations.append(pending_attestation) ``` ### Misc block post-processing From 393436992dd79fa5f31503e6c738c984575162aa Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 3 Nov 2019 12:06:19 -0800 Subject: [PATCH 018/111] Restructured shard blocks --- specs/core/1_new_shards.md | 82 +++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index cce39ab596..dd479394bc 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -64,9 +64,31 @@ This document describes the shard transition function (data layer only) and the | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | `DOMAIN_SHARD_LIGHT_CLIENT` | `192` | | +| `DOMAIN_SHARD_PROPOSAL` | `193` | | ## Containers +### `ShardBlockWrapper` + +```python +class ShardBlockWrapper(Container): + shard_parent_root: Hash + beacon_parent_root: Hash + slot: Slot + body: BytesN[SHARD_BLOCK_CHUNK_SIZE] + signature: BLSSignature +``` + +### `ShardSignedHeader` + +```python +class ShardSignedHeader(Container): + shard_parent_root: Hash + beacon_parent_root: Hash + slot: Slot + body_root: Hash +``` + ### `ShardState` ```python @@ -104,6 +126,8 @@ class ShardTransition(Container): shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate state roots shard_state_roots: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] + # Proposer signature aggregate + proposer_signature_aggregate: BLSSignature ``` ### `Attestation` @@ -322,23 +346,47 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: # Slot the attestation starts counting from start_slot = state.shard_next_slots[shard] + # Correct data root count offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) assert transition.start_slot == start_slot + def chunks_to_body_root(chunks): + return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) + + # Reonstruct shard headers + headers = [] + proposers = [] + shard_parent_root = state.shard_states[shard].latest_block_hash + for i in range(len(offset_slots)): + if any(transition.shard_data_roots): + headers.append(ShardSignedHeader( + shard_parent_root=shard_parent_root + parent_hash=get_block_root_at_slot(state, state.slot-1), + slot=offset_slots[i], + body_root=chunks_to_body_root(transition.shard_data_roots[i]) + )) + proposers.append(get_shard_proposer(state, shard, offset_slots[i])) + shard_parent_root = hash_tree_root(headers[-1]) + # Verify correct calculation of gas prices and slots and chunk roots prev_gasprice = state.shard_states[shard].gasprice for i in range(len(offset_slots)): shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] - block_length = transition.shard assert shard_state.gasprice == update_gasprice(prev_gasprice, block_length) assert shard_state.slot == offset_slots[i] assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE - filled_roots = chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) - assert shard_state.latest_block_hash == hash_tree_root(filled_roots) prev_gasprice = shard_state.gasprice + # Verify combined signature + assert bls_verify_multiple( + pubkeys=[state.validators[proposer].pubkey for proposer in proposers], + message_hashes=[hash_tree_root(header) for header in headers], + signature=proposer.proposer_signature_aggregate, + domain=DOMAIN_SHARD_PROPOSAL + ) + # Save updated state state.shard_states[shard] = transition.shard_states[-1] state.shard_states[shard].slot = state.slot - 1 @@ -450,29 +498,25 @@ def phase_1_epoch_transition(state): ### Fraud proofs -TODO. The intent is to have a single universal fraud proof type, which contains (i) an on-time attestation on shard `s` signing a set of `data_roots`, (ii) an index `i` of a particular data root to focus on, (iii) the full contents of the i'th data, (iii) a Merkle proof to the `shard_state_roots` in the parent block the attestation is referencing, and which then verifies that one of the two conditions is false: +TODO. The intent is to have a single universal fraud proof type, which contains the following parts: -* `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -* `execute_state_transition(shard, slot, attestation.shard_state_roots[i-1], hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != shard_state_roots[i]` (if `i=0` then instead use `parent.shard_state_roots[s][-1]`) +1. An on-time attestation on some `shard` signing a `ShardTransition` +2. An index `i` of a particular position to focus on +3. The `ShardTransition` itself +4. The full body of the block +5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing + +The proof verifies that one of the two conditions is false: + +1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` +2. `execute_state_transition(shard, slot, transition.shard_states[i-1].root, hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != transition.shard_states[i].root` (if `i=0` then instead use `parent.shard_states[shard][-1].root`) ## Shard state transition function ```python def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: - # Beginning of block data is the previous block hash - assert block_data[:32] == pre_state.latest_block_hash - assert block_data[32:64] == int_to_bytes8(slot) + b'\x00' * 24 - # Signature check - assert len(block_data) >= 160 - assert bls_verify( - pubkey=proposer_pubkey, - message_hash=hash_tree_root(block_data[:-96]), - signature=block_data[-96:], - domain=DOMAIN_SHARD_PROPOSER - ) # We will add something more substantive in phase 2 - length = len(block_data.rstrip(b'\x00')) - return ShardState(slot=slot, root=hash(pre_state + hash_tree_root(block_data)), gasprice=update_gasprice(pre_state, length), latest_block_hash = hash(block_data)) + return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) ``` ## Honest committee member behavior From d7234316164cb3cb4fbfeca4f88e346933df03ab Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 3 Nov 2019 15:49:50 -0800 Subject: [PATCH 019/111] Fixed pending attestation handling and added empty transition check --- specs/core/1_new_shards.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index dd479394bc..26ec88e3d6 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -404,6 +404,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S online_indices = get_online_indices(state) winners = set() for shard in range(ACTIVE_SHARDS): + success = False # All attestations in the block for this shard this_shard_attestations = [attestation for attestation in attestations if get_shard(state, attestation) == shard and attestation.data.slot == state.slot] # The committee for this shard @@ -428,8 +429,9 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S for shard_state, slot, length in zip(transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) winners.add((shard, shard_transition_root)) - for index in all_participants: - online_countdown[index] = ONLINE_PERIOD + success = True + if not success: + assert block.shard_transition == ShardTransition() for attestation in attestations: pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, @@ -454,6 +456,9 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock): for shard in range(MAX_SHARDS): if state.shard_states[shard].slot != state.slot - 1: assert block.shard_transition[shard] == ShardTransition() + for pending_attestation in state.current_epoch_attestations + state.previous_epoch_attestations: + for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): + online_countdown[index] = ONLINE_PERIOD ``` ### Light client processing From 03ab1d57858921524aca3cd77b2c78456346ee93 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 4 Nov 2019 08:50:09 -0800 Subject: [PATCH 020/111] A few cleanups --- specs/core/1_new_shards.md | 39 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 26ec88e3d6..fc8a19921c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -144,10 +144,8 @@ class Attestation(Container): ```python class IndexedAttestation(Container): - participants: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] - signature: BLSSignature + committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + attestation: Attestation ``` ### `CompactCommittee` @@ -232,8 +230,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - return IndexedAttestation(attesting_indices, attestation.data, attestation.custody_bits, attestation.signature) + committee = get_beacon_committee(beacon_state, attestation.data.slot, attestation.data.index) + return IndexedAttestation(committee, attestation) ``` ### `update_gasprice` @@ -259,18 +257,20 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe Check if ``indexed_attestation`` has valid indices and signature. """ - # Verify indices are sorted - if indexed_attestation.participants != sorted(indexed_attestation.participants): - return False - # Verify aggregate signature all_pubkeys = [] all_message_hashes = [] - for i, custody_bits in enumerate(indexed_attestation.custody_bits): - for participant, bit in zip(participants, custody_bits): - all_pubkeys.append(state.validators[participant].pubkey) - # Note: only 2N distinct message hashes - all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, bit)) + aggregation_bits = indexed_attestation.attestation.aggregation_bits + assert len(aggregation_bits) == len(indexed_attestation.committee) + for i, custody_bits in enumerate(indexed_attestation.attestation.custody_bits): + assert len(custody_bits) == len(indexed_attestation.committee) + for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): + if abit: + all_pubkeys.append(state.validators[participant].pubkey) + # Note: only 2N distinct message hashes + all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, cbit)) + else: + assert cbit == False return bls_verify_multiple( pubkeys=all_pubkeys, @@ -318,17 +318,13 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: proposer_index = get_beacon_proposer_index(state) # Signature check - committee = get_beacon_committee(state, get_current_epoch(state), shard) - for bits in attestation.custody_bits + [attestation.aggregation_bits]: - assert len(bits) == len(committee) assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Type 1: on-time attestations if data.custody_bits != []: # Correct slot assert data.slot == state.slot - # Slot the attestation starts counting from - start_slot = state.shard_next_slots[shard] # Correct data root count + start_slot = state.shard_next_slots[shard] offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] assert len(attestation.custody_bits) == len(offset_slots) # Correct parent block root @@ -379,7 +375,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE prev_gasprice = shard_state.gasprice - # Verify combined signature + # Verify combined proposer signature assert bls_verify_multiple( pubkeys=[state.validators[proposer].pubkey for proposer in proposers], message_hashes=[hash_tree_root(header) for header in headers], @@ -420,6 +416,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S if ( get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 + and success is False ): assert shard_transition_root == hash_tree_root(block.shard_transition) process_crosslink(state, shard, block.shard_transition) From 1fdd0332ccf2cb3a54828f3925c5bd209cf0df03 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Nov 2019 23:44:15 +0800 Subject: [PATCH 021/111] Update ToC --- specs/core/1_new_shards.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index fc8a19921c..c49eda5167 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -12,29 +12,36 @@ - [Configuration](#configuration) - [Misc](#misc) - [Containers](#containers) - - [Aliases](#aliases) + - [`ShardBlockWrapper`](#shardblockwrapper) + - [`ShardSignedHeader`](#shardsignedheader) + - [`ShardState`](#shardstate) - [`AttestationData`](#attestationdata) - - [`AttestationShardData`](#attestationsharddata) - - [`ReducedAttestationData`](#reducedattestationdata) + - [`ShardTransition`](#shardtransition) - [`Attestation`](#attestation) - - [`ReducedAttestation`](#reducedattestation) - [`IndexedAttestation`](#indexedattestation) - [`CompactCommittee`](#compactcommittee) - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) + - [`PendingAttestation`](#pendingattestation) - [Helpers](#helpers) - [`get_online_validators`](#get_online_validators) - [`pack_compact_validator`](#pack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`get_light_client_committee`](#get_light_client_committee) - [`get_indexed_attestation`](#get_indexed_attestation) + - [`update_gasprice`](#update_gasprice) - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) + - [`get_attestation_shard`](#get_attestation_shard) - [Beacon Chain Changes](#beacon-chain-changes) - [New beacon state fields](#new-beacon-state-fields) - [New beacon block data fields](#new-beacon-block-data-fields) - [Attestation processing](#attestation-processing) + - [`validate_attestation`](#validate_attestation) + - [`apply_shard_transition`](#apply_shard_transition) + - [`process_attestations`](#process_attestations) + - [Misc block post-processing](#misc-block-post-processing) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) - - [Fraud proofs](#fraud-proofs) + - [Fraud proofs](#fraud-proofs) - [Shard state transition function](#shard-state-transition-function) - [Honest committee member behavior](#honest-committee-member-behavior) @@ -306,9 +313,9 @@ def get_shard(state: BeaconState, attestation: Attestation): light_client_signature: BLSSignature ``` -## Attestation processing +### Attestation processing -### `validate_attestation` +#### `validate_attestation` ```python def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: @@ -336,7 +343,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: assert len(attestation.custody_bits) == 0 ``` -### `apply_shard_transition` +#### `apply_shard_transition` ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: @@ -388,7 +395,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr state.shard_states[shard].slot = state.slot - 1 ``` -### `process_attestations` +#### `process_attestations` ```python def process_attestations(state: BeaconState, block: BeaconBlock, attestations: Sequence[Attestation]) -> None: @@ -498,7 +505,7 @@ def phase_1_epoch_transition(state): state.next_light_committee = committee_to_compact_committee(state, new_committee) ``` -### Fraud proofs +## Fraud proofs TODO. The intent is to have a single universal fraud proof type, which contains the following parts: From 462d05d6d8f5be39e2f426e702c9fc557ad757cd Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:31:59 -0800 Subject: [PATCH 022/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index c49eda5167..c27b4d0c97 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -61,7 +61,7 @@ This document describes the shard transition function (data layer only) and the | `ACTIVE_SHARDS` | `2**6` (= 64) | | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | -| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~29 hours | +| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | | `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | | `BLOCK_SIZE_TARGET` | `3 * 2**16` (= 196,608) | | From 213032e51f1a929dc96e3144cd81de65907feaf8 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:32:18 -0800 Subject: [PATCH 023/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index c27b4d0c97..c3e6ebc53b 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -290,7 +290,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ### `get_attestation_shard` ```python -def get_shard(state: BeaconState, attestation: Attestation): +def get_shard(state: BeaconState, attestation: Attestation) -> Shard: return (attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS ``` From d3f9bfe68d440b0c49161bbf81e121549bd9ee24 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:32:49 -0800 Subject: [PATCH 024/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index c3e6ebc53b..de436004e4 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -275,7 +275,9 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe if abit: all_pubkeys.append(state.validators[participant].pubkey) # Note: only 2N distinct message hashes - all_message_hashes.append(AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, cbit)) + all_message_hashes.append(hash_tree_root( + AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, cbit) + )) else: assert cbit == False From d59e87f8764dd9065913da960abff8e1d3fecb7a Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:33:26 -0800 Subject: [PATCH 025/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index de436004e4..477b95a103 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -320,7 +320,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: #### `validate_attestation` ```python -def validate_attestation(state: BeaconState, attestation: Attestation) -> bool: +def validate_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data assert data.index < ACTIVE_SHARDS shard = get_shard(state, attestation) From bcd06d88ef85bd11d93237fa7773cc5389a7f2b5 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:33:41 -0800 Subject: [PATCH 026/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 477b95a103..03b574cb2c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -367,7 +367,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for i in range(len(offset_slots)): if any(transition.shard_data_roots): headers.append(ShardSignedHeader( - shard_parent_root=shard_parent_root + shard_parent_root=shard_parent_root, parent_hash=get_block_root_at_slot(state, state.slot-1), slot=offset_slots[i], body_root=chunks_to_body_root(transition.shard_data_roots[i]) From 9d3a230974c1df44cb29cadfa207d57cf20f34cf Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:34:04 -0800 Subject: [PATCH 027/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 03b574cb2c..d105782656 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -404,7 +404,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S pending_attestations = [] # Basic validation for attestation in attestations: - assert validate_attestation(state, attestation) + validate_attestation(state, attestation) # Process crosslinks online_indices = get_online_indices(state) winners = set() From 9810a3d2606c41ad56f5ae26f5260bdb0c564bec Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:34:34 -0800 Subject: [PATCH 028/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index d105782656..8e1ede1084 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -432,7 +432,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) - for shard_state, slot, length in zip(transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): + for shard_state, slot, length in zip(block.shard_transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) winners.add((shard, shard_transition_root)) success = True From 4d1e9e15c67f6a8d250999d67695be55137fdc3b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 10:34:48 -0800 Subject: [PATCH 029/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 8e1ede1084..3659cc5995 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -293,7 +293,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: - return (attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS + return Shard((attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS) ``` ## Beacon Chain Changes From 8a6bd201f9f341700d0aba355b623eb7261aedfe Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:15:04 -0800 Subject: [PATCH 030/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 3659cc5995..fa8397bade 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -443,7 +443,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S aggregation_bits=attestation.aggregation_bits, data=attestation.data, inclusion_delay=state.slot - data.slot, - crosslink_success=(attestation.shard, attestation.shard_transition_root) in winners and attestation.data.slot == state.slot, + crosslink_success=(get_shard(state, attestation), attestation.shard_transition_root) in winners and attestation.data.slot == state.slot, proposer_index=proposer_index ) if attestation.data.target.epoch == get_current_epoch(state): From 41be60ee6e4b782d54f0de58c69cc47b913669fc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:15:47 -0800 Subject: [PATCH 031/111] Update specs/core/1_new_shards.md --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index fa8397bade..074e06db7b 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -428,7 +428,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S and success is False ): assert shard_transition_root == hash_tree_root(block.shard_transition) - process_crosslink(state, shard, block.shard_transition) + apply_shard_transition(state, shard, block.shard_transition) # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) From 793c8616cd00481dd2b44978dd577ab9cb17b148 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:16:07 -0800 Subject: [PATCH 032/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 074e06db7b..99cf82ce45 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -457,7 +457,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S ### Misc block post-processing ```python -def misc_block_post_process(state: BeaconState, block: BeaconBlock): +def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(MAX_SHARDS): if state.shard_states[shard].slot != state.slot - 1: From bf13757722654017d3874b57bc3ed7a42fe01dcf Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:16:25 -0800 Subject: [PATCH 033/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 99cf82ce45..4ee5025850 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -464,7 +464,7 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: assert block.shard_transition[shard] == ShardTransition() for pending_attestation in state.current_epoch_attestations + state.previous_epoch_attestations: for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): - online_countdown[index] = ONLINE_PERIOD + state.online_countdown[index] = ONLINE_PERIOD ``` ### Light client processing From 0ddc0ba527efc199847f3d3a8285751c06bf39d3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:16:36 -0800 Subject: [PATCH 034/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 4ee5025850..bd25545f2c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -470,7 +470,7 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: ### Light client processing ```python -def process_light_client_signatures(state: BeaconState, block: BeaconBlock): +def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) assert len(block.light_client_signature_bitfield) == len(committee) tot_reward = 0 From 822ceade5835261a9b284da79311466545b4f071 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:16:56 -0800 Subject: [PATCH 035/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index bd25545f2c..f6a1ece72c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -473,7 +473,7 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) assert len(block.light_client_signature_bitfield) == len(committee) - tot_reward = 0 + total_reward = Gwei(0) signer_keys = [] for i, bit in enumerate(block.light_client_signature_bitfield): if bit: From a749dd9a7789f3493992a7bb84251af90f63d71b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:17:07 -0800 Subject: [PATCH 036/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index f6a1ece72c..aa3e4f1888 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -479,7 +479,7 @@ def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> N if bit: signer_keys.append(state.validators[committee[i]].pubkey) increase_balance(state, committee[i], get_base_reward(state, committee[i])) - tot_reward += get_base_reward(state, committee[i]) + total_reward += get_base_reward(state, committee[i]) increase_balance(state, get_beacon_proposer_index(state), tot_reward // PROPOSER_REWARD_COEFFICIENT) From c9cc110ab1d6a334db23d3a8aa1afc53baf553ad Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:17:27 -0800 Subject: [PATCH 037/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index aa3e4f1888..0730882dd7 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -481,7 +481,7 @@ def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> N increase_balance(state, committee[i], get_base_reward(state, committee[i])) total_reward += get_base_reward(state, committee[i]) - increase_balance(state, get_beacon_proposer_index(state), tot_reward // PROPOSER_REWARD_COEFFICIENT) + increase_balance(state, get_beacon_proposer_index(state), total_reward // PROPOSER_REWARD_COEFFICIENT) assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), From 2ea8eb9cc3a93885a7058619efa88899c1ea6993 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 5 Nov 2019 12:17:50 -0800 Subject: [PATCH 038/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 0730882dd7..70a12d2189 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -494,7 +494,7 @@ def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> N ### Epoch transition ```python -def phase_1_epoch_transition(state): +def phase_1_epoch_transition(state: BeaconState) -> None: # Slowly remove validators from the "online" set if they do not show up for index in range(len(state.validators)): if state.online_countdown[index] != 0: From ff3bd725a36e9ac6400c74c640060adb74b92371 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Nov 2019 15:53:01 -0500 Subject: [PATCH 039/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 70a12d2189..5ec4a7e194 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -132,7 +132,7 @@ class ShardTransition(Container): # Shard data roots shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate state roots - shard_state_roots: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] + shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Proposer signature aggregate proposer_signature_aggregate: BLSSignature ``` From 8cb404d92663fa036b129f218ab6c850f153cac4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Nov 2019 15:53:17 -0500 Subject: [PATCH 040/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 5ec4a7e194..149aee358c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -388,7 +388,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr assert bls_verify_multiple( pubkeys=[state.validators[proposer].pubkey for proposer in proposers], message_hashes=[hash_tree_root(header) for header in headers], - signature=proposer.proposer_signature_aggregate, + signature=transition.proposer_signature_aggregate, domain=DOMAIN_SHARD_PROPOSAL ) From e66b4e06eff60d4f4f902b88f8fb78bb468a3213 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Nov 2019 15:53:41 -0500 Subject: [PATCH 041/111] Update specs/core/1_new_shards.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 149aee358c..f9fbfde766 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -340,7 +340,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Type 2: delayed attestations else: - assert state.slot - slot_to_epoch(data.slot) < EPOCH_LENGTH + assert state.slot < data.slot + SLOTS_PER_EPOCH assert data.shard_transition_root == Hash() assert len(attestation.custody_bits) == 0 ``` From b6680d28bb467abb1f83c04d52ec013e64e3ca67 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Nov 2019 15:54:30 -0500 Subject: [PATCH 042/111] Update specs/core/1_new_shards.md Co-Authored-By: Danny Ryan --- specs/core/1_new_shards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index f9fbfde766..01701f5145 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -437,7 +437,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S winners.add((shard, shard_transition_root)) success = True if not success: - assert block.shard_transition == ShardTransition() + assert block.shard_transitions[shard] == ShardTransition() for attestation in attestations: pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, From 8f34a0788dd786f615204e7ba0b5d9565d2b1906 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Wed, 6 Nov 2019 14:19:00 -0800 Subject: [PATCH 043/111] Some initial changes --- specs/core/1_new_shards.md | 51 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 01701f5145..48a575121b 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Containers](#containers) - [`ShardBlockWrapper`](#shardblockwrapper) - - [`ShardSignedHeader`](#shardsignedheader) + - [`ShardSignableHeader`](#shardsignedheader) - [`ShardState`](#shardstate) - [`AttestationData`](#attestationdata) - [`ShardTransition`](#shardtransition) @@ -28,7 +28,7 @@ - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`get_light_client_committee`](#get_light_client_committee) - [`get_indexed_attestation`](#get_indexed_attestation) - - [`update_gasprice`](#update_gasprice) + - [`get_updated_gasprice`](#get_updated_gasprice) - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - [`get_attestation_shard`](#get_attestation_shard) - [Beacon Chain Changes](#beacon-chain-changes) @@ -77,6 +77,8 @@ This document describes the shard transition function (data layer only) and the ### `ShardBlockWrapper` +_Wrapper for being broadcasted over the network._ + ```python class ShardBlockWrapper(Container): shard_parent_root: Hash @@ -86,10 +88,10 @@ class ShardBlockWrapper(Container): signature: BLSSignature ``` -### `ShardSignedHeader` +### `ShardSignableHeader` ```python -class ShardSignedHeader(Container): +class ShardSignableHeader(Container): shard_parent_root: Hash beacon_parent_root: Hash slot: Slot @@ -102,7 +104,7 @@ class ShardSignedHeader(Container): class ShardState(Container): slot: Slot gasprice: Gwei - root: Hash + data: Hash latest_block_hash: Hash ``` @@ -131,7 +133,7 @@ class ShardTransition(Container): shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] - # Intermediate state roots + # Intermediate shard states shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Proposer signature aggregate proposer_signature_aggregate: BLSSignature @@ -241,10 +243,10 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) return IndexedAttestation(committee, attestation) ``` -### `update_gasprice` +### `get_updated_gasprice` ```python -def update_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: +def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: if length > BLOCK_SIZE_TARGET: delta = prev_gasprice * (length - BLOCK_SIZE_TARGET) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT return min(prev_gasprice + delta, MAX_GASPRICE) @@ -289,13 +291,20 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ) ``` -### `get_attestation_shard` +### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: return Shard((attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS) ``` +### `get_offset_slots` + +```python +def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: + return [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] +``` + ## Beacon Chain Changes ### New beacon state fields @@ -333,16 +342,13 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Correct slot assert data.slot == state.slot # Correct data root count - start_slot = state.shard_next_slots[shard] - offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] - assert len(attestation.custody_bits) == len(offset_slots) + assert len(attestation.custody_bits) == len(get_offset_slots(state, state.shard_next_slots[shard])) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Type 2: delayed attestations else: assert state.slot < data.slot + SLOTS_PER_EPOCH assert data.shard_transition_root == Hash() - assert len(attestation.custody_bits) == 0 ``` #### `apply_shard_transition` @@ -353,7 +359,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr start_slot = state.shard_next_slots[shard] # Correct data root count - offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] + offset_slots = get_offset_slots(state, start_slot) assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) assert transition.start_slot == start_slot @@ -366,7 +372,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr shard_parent_root = state.shard_states[shard].latest_block_hash for i in range(len(offset_slots)): if any(transition.shard_data_roots): - headers.append(ShardSignedHeader( + headers.append(ShardSignableHeader( shard_parent_root=shard_parent_root, parent_hash=get_block_root_at_slot(state, state.slot-1), slot=offset_slots[i], @@ -379,7 +385,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr prev_gasprice = state.shard_states[shard].gasprice for i in range(len(offset_slots)): shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] - assert shard_state.gasprice == update_gasprice(prev_gasprice, block_length) + assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) assert shard_state.slot == offset_slots[i] assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE prev_gasprice = shard_state.gasprice @@ -462,9 +468,6 @@ def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: for shard in range(MAX_SHARDS): if state.shard_states[shard].slot != state.slot - 1: assert block.shard_transition[shard] == ShardTransition() - for pending_attestation in state.current_epoch_attestations + state.previous_epoch_attestations: - for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): - state.online_countdown[index] = ONLINE_PERIOD ``` ### Light client processing @@ -505,6 +508,11 @@ def phase_1_epoch_transition(state: BeaconState) -> None: state.current_light_committee = state.next_light_committee new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) state.next_light_committee = committee_to_compact_committee(state, new_committee) + + # Process pending attestations + for pending_attestation in state.current_epoch_attestations + state.previous_epoch_attestations: + for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): + state.online_countdown[index] = ONLINE_PERIOD ``` ## Fraud proofs @@ -520,7 +528,7 @@ TODO. The intent is to have a single universal fraud proof type, which contains The proof verifies that one of the two conditions is false: 1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -2. `execute_state_transition(shard, slot, transition.shard_states[i-1].root, hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != transition.shard_states[i].root` (if `i=0` then instead use `parent.shard_states[shard][-1].root`) +2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) ## Shard state transition function @@ -535,8 +543,7 @@ def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_b Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on. Three seconds into slot `slot`, run the following procedure: * Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. -* Let `offset_slots = [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot]` -* For `slot in offset_slots`, do the following: +* For `slot in get_offset_slots(state, start_slot)`, do the following: * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` * If `len(choices) == 1`, do `proposals.append(choices[0])` From 7b4a6e8307037375fe4023fd2acf986686c023cb Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 7 Nov 2019 05:21:36 -0800 Subject: [PATCH 044/111] Simplified gasprice update --- specs/core/1_new_shards.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index 48a575121b..c9cbad161c 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -69,6 +69,7 @@ This document describes the shard transition function (data layer only) and the | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | +| `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | `DOMAIN_SHARD_LIGHT_CLIENT` | `192` | | | `DOMAIN_SHARD_PROPOSAL` | `193` | | @@ -252,10 +253,7 @@ def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: return min(prev_gasprice + delta, MAX_GASPRICE) else: delta = prev_gasprice * (BLOCK_SIZE_TARGET - length) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT - if delta > prev_gasprice - GASPRICE_ADJUSTMENT_COEFFICIENT: - return GASPRICE_ADJUSTMENT_COEFFICIENT - else: - return prev_gasprice - delta + return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` ### `is_valid_indexed_attestation` From 4d6dcd15ba29f271e6d00a16080c19ed0e4a1fb7 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 12 Nov 2019 05:27:34 -0800 Subject: [PATCH 045/111] Some fixes --- specs/core/1_new_shards.md | 41 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index c9cbad161c..b3427afe04 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -18,7 +18,7 @@ - [`AttestationData`](#attestationdata) - [`ShardTransition`](#shardtransition) - [`Attestation`](#attestation) - - [`IndexedAttestation`](#indexedattestation) + - [`AttestationAndCommittee`](#attestationandcommittee) - [`CompactCommittee`](#compactcommittee) - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - [`PendingAttestation`](#pendingattestation) @@ -106,7 +106,7 @@ class ShardState(Container): slot: Slot gasprice: Gwei data: Hash - latest_block_hash: Hash + latest_block_root: Hash ``` ### `AttestationData` @@ -120,6 +120,8 @@ class AttestationData(Container): # FFG vote source: Checkpoint target: Checkpoint + # Current-slot shard block root + head_shard_root: Hash # Shard transition root shard_transition_root: Hash ``` @@ -150,10 +152,10 @@ class Attestation(Container): signature: BLSSignature ``` -### `IndexedAttestation` +### `AttestationAndCommittee` ```python -class IndexedAttestation(Container): +class AttestationAndCommittee(Container): committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] attestation: Attestation ``` @@ -239,9 +241,9 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ### `get_indexed_attestation` ```python -def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: +def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> AttestationAndCommittee: committee = get_beacon_committee(beacon_state, attestation.data.slot, attestation.data.index) - return IndexedAttestation(committee, attestation) + return AttestationAndCommittee(committee, attestation) ``` ### `get_updated_gasprice` @@ -259,7 +261,7 @@ def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: ### `is_valid_indexed_attestation` ```python -def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: AttestationAndCommittee) -> bool: """ Check if ``indexed_attestation`` has valid indices and signature. """ @@ -303,6 +305,13 @@ def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: return [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] ``` +### `chunks_to_body_root` + +```python +def chunks_to_body_root(chunks): + return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) +``` + ## Beacon Chain Changes ### New beacon state fields @@ -336,7 +345,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Signature check assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Type 1: on-time attestations - if data.custody_bits != []: + if attestation.custody_bits != []: # Correct slot assert data.slot == state.slot # Correct data root count @@ -345,7 +354,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Type 2: delayed attestations else: - assert state.slot < data.slot + SLOTS_PER_EPOCH + assert state.slot - compute_start_slot_at_epoch(slot_to_epoch(data.slot)) < EPOCH_LENGTH assert data.shard_transition_root == Hash() ``` @@ -361,13 +370,10 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) assert transition.start_slot == start_slot - def chunks_to_body_root(chunks): - return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) - # Reonstruct shard headers headers = [] proposers = [] - shard_parent_root = state.shard_states[shard].latest_block_hash + shard_parent_root = state.shard_states[shard].latest_block_root for i in range(len(offset_slots)): if any(transition.shard_data_roots): headers.append(ShardSignableHeader( @@ -431,7 +437,10 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 and success is False ): + # Attestation <-> shard transition consistency assert shard_transition_root == hash_tree_root(block.shard_transition) + assert attestation.data.head_shard_root == chunks_to_body_root(block.shard_transition.shard_data_roots[-1]) + # Apply transition apply_shard_transition(state, shard, block.shard_transition) # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) @@ -461,7 +470,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S ### Misc block post-processing ```python -def misc_block_post_process(state: BeaconState, block: BeaconBlock) -> None: +def verify_shard_transition_false_positives(state: BeaconState, block: BeaconBlock) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(MAX_SHARDS): if state.shard_states[shard].slot != state.slot - 1: @@ -538,7 +547,7 @@ def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_b ## Honest committee member behavior -Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on. Three seconds into slot `slot`, run the following procedure: +Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `slot`, run the following procedure: * Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. * For `slot in get_offset_slots(state, start_slot)`, do the following: @@ -546,6 +555,6 @@ Suppose you are a committee member on shard `shard` at slot `current_slot`. Let * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` * If `len(choices) == 1`, do `proposals.append(choices[0])` * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. - * Set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. + * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. From ef936b94fef5f471a7092e741ed0b7c00a225f93 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 12 Nov 2019 06:13:47 -0800 Subject: [PATCH 046/111] Added get_shard_proposer_index --- specs/core/1_new_shards.md | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/specs/core/1_new_shards.md b/specs/core/1_new_shards.md index b3427afe04..df543f2488 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_new_shards.md @@ -62,6 +62,7 @@ This document describes the shard transition function (data layer only) and the | `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | +| `SHARD_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | | `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | | `BLOCK_SIZE_TARGET` | `3 * 2**16` (= 196,608) | | @@ -72,6 +73,7 @@ This document describes the shard transition function (data layer only) and the | `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | `DOMAIN_SHARD_LIGHT_CLIENT` | `192` | | +| `DOMAIN_SHARD_COMMITTEE` | `192` | | | `DOMAIN_SHARD_PROPOSAL` | `193` | | ## Containers @@ -226,6 +228,27 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` +### `get_shard_committee` + +```python +def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: + source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD + if source_epoch > 0: + source_epoch -= SHARD_COMMITTEE_PERIOD + active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) + seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) + return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS) +``` + +### `get_shard_proposer_index` + +```python +def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: + committee = get_shard_committee(beacon_state, slot_to_epoch(slot), shard) + r = bytes_to_int(get_seed(beacon_state, get_current_epoch(state), DOMAIN_SHARD_COMMITTEE)[:8]) + return committee[r % len(committee)] +``` + ### `get_light_client_committee` ```python @@ -382,7 +405,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr slot=offset_slots[i], body_root=chunks_to_body_root(transition.shard_data_roots[i]) )) - proposers.append(get_shard_proposer(state, shard, offset_slots[i])) + proposers.append(get_shard_proposer_index(state, shard, offset_slots[i])) shard_parent_root = hash_tree_root(headers[-1]) # Verify correct calculation of gas prices and slots and chunk roots @@ -446,7 +469,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) for shard_state, slot, length in zip(block.shard_transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): - decrease_balance(state, get_shard_proposer(state, shard, slot), shard_state.gasprice * length) + decrease_balance(state, get_shard_proposer_index(state, shard, slot), shard_state.gasprice * length) winners.add((shard, shard_transition_root)) success = True if not success: @@ -535,7 +558,7 @@ TODO. The intent is to have a single universal fraud proof type, which contains The proof verifies that one of the two conditions is false: 1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) +2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer_index(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) ## Shard state transition function @@ -551,10 +574,10 @@ Suppose you are a committee member on shard `shard` at slot `current_slot`. Let * Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. * For `slot in get_offset_slots(state, start_slot)`, do the following: - * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. + * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` * If `len(choices) == 1`, do `proposals.append(choices[0])` * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. - * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. + * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. From a33e01dab2d5cab9a6aab7188c3b5456fa23aaa0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 21:11:42 +0100 Subject: [PATCH 047/111] Start making phase1 work in pyspec again --- specs/core/0_beacon-chain.md | 21 +- .../{1_new_shards.md => 1_beacon-chain.md} | 325 +++++++++++------- specs/core/1_custody-game.md | 54 +-- specs/core/1_fraudproofs.md | 52 +++ specs/{core => old}/1_beacon-chain-misc.md | 0 specs/{core => old}/1_shard-data-chains.md | 0 6 files changed, 257 insertions(+), 195 deletions(-) rename specs/core/{1_new_shards.md => 1_beacon-chain.md} (74%) create mode 100644 specs/core/1_fraudproofs.md rename specs/{core => old}/1_beacon-chain-misc.md (100%) rename specs/{core => old}/1_shard-data-chains.md (100%) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index fca21994f7..e8aa53f13b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1129,19 +1129,13 @@ def process_slot(state: BeaconState) -> None: ### Epoch processing -*Note*: The `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. - ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) - # @process_reveal_deadlines - # @process_challenge_deadlines process_slashings(state) - # @update_period_committee process_final_updates(state) - # @after_process_final_updates ``` #### Helper functions @@ -1425,16 +1419,11 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - for operations, function in ( - (body.proposer_slashings, process_proposer_slashing), - (body.attester_slashings, process_attester_slashing), - (body.attestations, process_attestation), - (body.deposits, process_deposit), - (body.voluntary_exits, process_voluntary_exit), - # @process_shard_receipt_proofs - ): - for operation in operations: - function(state, operation) + process_operations(body.proposer_slashings, process_proposer_slashing) + process_operations(body.attester_slashings, process_attester_slashing) + process_operations(body.attestations, process_attestations) + process_operations(body.deposits, process_deposit) + process_operations(body.voluntary_exits, process_voluntary_exit) ``` ##### Proposer slashings diff --git a/specs/core/1_new_shards.md b/specs/core/1_beacon-chain.md similarity index 74% rename from specs/core/1_new_shards.md rename to specs/core/1_beacon-chain.md index df543f2488..04df84e866 100644 --- a/specs/core/1_new_shards.md +++ b/specs/core/1_beacon-chain.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data +# Ethereum 2.0 Phase 1 -- The Beacon Chain for Shards **Notice**: This document is a work-in-progress for researchers and implementers. @@ -6,53 +6,20 @@ -- [Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data](#ethereum-20-phase-1----crosslinks-and-shard-data) - - [Table of contents](#table-of-contents) - - [Introduction](#introduction) - - [Configuration](#configuration) - - [Misc](#misc) - - [Containers](#containers) - - [`ShardBlockWrapper`](#shardblockwrapper) - - [`ShardSignableHeader`](#shardsignedheader) - - [`ShardState`](#shardstate) - - [`AttestationData`](#attestationdata) - - [`ShardTransition`](#shardtransition) - - [`Attestation`](#attestation) - - [`AttestationAndCommittee`](#attestationandcommittee) - - [`CompactCommittee`](#compactcommittee) - - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - - [`PendingAttestation`](#pendingattestation) - - [Helpers](#helpers) - - [`get_online_validators`](#get_online_validators) - - [`pack_compact_validator`](#pack_compact_validator) - - [`committee_to_compact_committee`](#committee_to_compact_committee) - - [`get_light_client_committee`](#get_light_client_committee) - - [`get_indexed_attestation`](#get_indexed_attestation) - - [`get_updated_gasprice`](#get_updated_gasprice) - - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - - [`get_attestation_shard`](#get_attestation_shard) - - [Beacon Chain Changes](#beacon-chain-changes) - - [New beacon state fields](#new-beacon-state-fields) - - [New beacon block data fields](#new-beacon-block-data-fields) - - [Attestation processing](#attestation-processing) - - [`validate_attestation`](#validate_attestation) - - [`apply_shard_transition`](#apply_shard_transition) - - [`process_attestations`](#process_attestations) - - [Misc block post-processing](#misc-block-post-processing) - - [Light client processing](#light-client-processing) - - [Epoch transition](#epoch-transition) - - [Fraud proofs](#fraud-proofs) - - [Shard state transition function](#shard-state-transition-function) - - [Honest committee member behavior](#honest-committee-member-behavior) +TODO ## Introduction -This document describes the shard transition function (data layer only) and the shard fork choice rule as part of Phase 1 of Ethereum 2.0. +This document describes the extensions made to the Phase 0 design of The Beacon Chain + to facilitate the new shards as part of Phase 1 of Eth2. ## Configuration +Configuration is not namespaced. Instead it is strictly an extension; + no constants of phase 0 change, but new constants are adopted for changing behaviors. + ### Misc | Name | Value | Unit | Duration | @@ -111,7 +78,7 @@ class ShardState(Container): latest_block_root: Hash ``` -### `AttestationData` +### New `AttestationData` ```python class AttestationData(Container): @@ -144,7 +111,7 @@ class ShardTransition(Container): proposer_signature_aggregate: BLSSignature ``` -### `Attestation` +### New `Attestation` ```python class Attestation(Container): @@ -179,7 +146,7 @@ class AttestationCustodyBitWrapper(Container): bit: bool ``` -### `PendingAttestation` +### New `PendingAttestation` ```python class PendingAttestation(Container): @@ -190,17 +157,112 @@ class PendingAttestation(Container): crosslink_success: bool ``` -## Helpers +### New extended `Validator` + +```python +class Validator(Container): + pubkey: BLSPubkey + withdrawal_credentials: Hash # Commitment to pubkey for withdrawals + effective_balance: Gwei # Balance at stake + slashed: boolean + # Status epochs + activation_eligibility_epoch: Epoch # When criteria for activation were met + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw funds + + # TODO: older pre-proposal custody field additions, keep this? + # + # next_custody_secret_to_reveal is initialised to the custody period + # (of the particular validator) in which the validator is activated + # = get_custody_period_for_validator(...) + next_custody_secret_to_reveal: uint64 + max_reveal_lateness: Epoch +``` -### `get_online_validators` + +### New extended `BeaconBlock` ```python -def get_online_indices(state: BeaconState) -> Set[ValidatorIndex]: - active_validators = get_active_validator_indices(state, get_current_epoch(state)) - return set([i for i in active_validators if state.online_countdown[i] != 0]) +class BeaconBlock(phase0.BeaconBlock): + slot: Slot + parent_root: Hash + state_root: Hash + body: BeaconBlockBody + shard_transitions: Vector[ShardTransition, MAX_SHARDS] + light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_signature: BLSSignature + + # TODO: older pre-proposal custody field additions, keep this? + custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] + custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] + custody_responses: List[CustodyResponse, PLACEHOLDER] + custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] + + signature: BLSSignature +``` + +### New extended `BeaconState` + +```python +class BeaconState(phase0.BeaconState): + # Versioning + genesis_time: uint64 + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Hash, HISTORICAL_ROOTS_LIMIT] + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Attestations + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + # Phase 1 + shard_states: Vector[ShardState, MAX_SHARDS] + online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] + current_light_committee: CompactCommittee + next_light_committee: CompactCommittee + + # TODO older pre-proposal custody field additions, keep this? + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] + custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] + custody_challenge_index: uint64 + # Future derived secrets already exposed; contains the indices of the exposed validator + # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS + exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` -### `pack_compact_validator` +## Helper functions + +### Crypto + +#### `bls_verify_multiple` + +`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). + + +### Misc + +#### `pack_compact_validator` ```python def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: @@ -212,7 +274,7 @@ def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int return (index << 16) + (slashed << 15) + balance_in_increments ``` -### `committee_to_compact_committee` +#### `committee_to_compact_committee` ```python def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: @@ -228,7 +290,24 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` -### `get_shard_committee` +#### `chunks_to_body_root` + +```python +def chunks_to_body_root(chunks): + return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) +``` + +### Beacon state accessors + +#### `get_online_validators` + +```python +def get_online_indices(state: BeaconState) -> Set[ValidatorIndex]: + active_validators = get_active_validator_indices(state, get_current_epoch(state)) + return set([i for i in active_validators if state.online_countdown[i] != 0]) +``` + +#### `get_shard_committee` ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: @@ -240,7 +319,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS) ``` -### `get_shard_proposer_index` +#### `get_shard_proposer_index` ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: @@ -249,7 +328,7 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard return committee[r % len(committee)] ``` -### `get_light_client_committee` +#### `get_light_client_committee` ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: @@ -261,7 +340,7 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS)[:TARGET_COMMITTEE_SIZE] ``` -### `get_indexed_attestation` +#### `get_indexed_attestation` ```python def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> AttestationAndCommittee: @@ -269,7 +348,7 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) return AttestationAndCommittee(committee, attestation) ``` -### `get_updated_gasprice` +#### `get_updated_gasprice` ```python def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: @@ -281,7 +360,24 @@ def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` -### `is_valid_indexed_attestation` +#### `get_shard` + +```python +def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + return Shard((attestation.data.index + get_start_shard(state, attestation.data.slot)) % ACTIVE_SHARDS) +``` + +#### `get_offset_slots` + +```python +def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: + return [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] +``` + + +### Predicates + +#### `is_valid_indexed_attestation` ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: AttestationAndCommittee) -> bool: @@ -314,49 +410,46 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Attest ) ``` -### `get_shard` - -```python -def get_shard(state: BeaconState, attestation: Attestation) -> Shard: - return Shard((attestation.data.index + get_start_shard(state, data.slot)) % ACTIVE_SHARDS) -``` -### `get_offset_slots` +### Block processing ```python -def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: - return [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] -``` - -### `chunks_to_body_root` - -```python -def chunks_to_body_root(chunks): - return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) + verify_shard_transition_false_positives(state, block) + process_light_client_signatures(state, block) + process_operations(state, block.body) ``` -## Beacon Chain Changes -### New beacon state fields +#### Operations ```python - shard_states: Vector[ShardState, MAX_SHARDS] - online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] - current_light_committee: CompactCommittee - next_light_committee: CompactCommittee -``` +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def process_operations(operations, fn): + for operation in operations: + fn(state, operation) + + process_operations(body.proposer_slashings, process_proposer_slashing) + process_operations(body.attester_slashings, process_attester_slashing) -### New beacon block data fields + # New attestation processing + process_attestations(state, block, body.attestations) -```python - shard_transitions: Vector[ShardTransition, MAX_SHARDS] - light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] - light_client_signature: BLSSignature + process_operations(body.deposits, process_deposit) + process_operations(body.voluntary_exits, process_voluntary_exit) + + # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` -### Attestation processing +##### New Attestation processing -#### `validate_attestation` +###### `validate_attestation` ```python def validate_attestation(state: BeaconState, attestation: Attestation) -> None: @@ -381,7 +474,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.shard_transition_root == Hash() ``` -#### `apply_shard_transition` +###### `apply_shard_transition` ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: @@ -430,7 +523,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr state.shard_states[shard].slot = state.slot - 1 ``` -#### `process_attestations` +###### `process_attestations` ```python def process_attestations(state: BeaconState, block: BeaconBlock, attestations: Sequence[Attestation]) -> None: @@ -490,7 +583,7 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S state.previous_epoch_attestations.append(pending_attestation) ``` -### Misc block post-processing +#### Shard transition false positives ```python def verify_shard_transition_false_positives(state: BeaconState, block: BeaconBlock) -> None: @@ -500,7 +593,7 @@ def verify_shard_transition_false_positives(state: BeaconState, block: BeaconBlo assert block.shard_transition[shard] == ShardTransition() ``` -### Light client processing +#### Light client processing ```python def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> None: @@ -524,10 +617,23 @@ def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> N ) ``` + ### Epoch transition +This epoch transition overrides the phase0 epoch transition: + ```python -def phase_1_epoch_transition(state: BeaconState) -> None: +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + # TODO process_reveal_deadlines + # TODO process_challenge_deadlines + process_slashings(state) + # TODO update_period_committee + process_final_updates(state) + # TODO process_custody_final_updates + # Slowly remove validators from the "online" set if they do not show up for index in range(len(state.validators)): if state.online_countdown[index] != 0: @@ -544,40 +650,3 @@ def phase_1_epoch_transition(state: BeaconState) -> None: for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): state.online_countdown[index] = ONLINE_PERIOD ``` - -## Fraud proofs - -TODO. The intent is to have a single universal fraud proof type, which contains the following parts: - -1. An on-time attestation on some `shard` signing a `ShardTransition` -2. An index `i` of a particular position to focus on -3. The `ShardTransition` itself -4. The full body of the block -5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing - -The proof verifies that one of the two conditions is false: - -1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer_index(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) - -## Shard state transition function - -```python -def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: - # We will add something more substantive in phase 2 - return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) -``` - -## Honest committee member behavior - -Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `slot`, run the following procedure: - -* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. -* For `slot in get_offset_slots(state, start_slot)`, do the following: - * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. - * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` - * If `len(choices) == 1`, do `proposals.append(choices[0])` - * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. - * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. - -Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index bb3a9b32b3..087dcdbf9a 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -231,45 +231,6 @@ class EarlyDerivedSecretReveal(Container): mask: Bytes32 ``` -### Phase 0 container updates - -Add the following fields to the end of the specified container objects. Fields with underlying type `uint64` are initialized to `0` and list fields are initialized to `[]`. - -#### `Validator` - -```python -class Validator(Container): - # next_custody_secret_to_reveal is initialised to the custody period - # (of the particular validator) in which the validator is activated - # = get_custody_period_for_validator(...) - next_custody_secret_to_reveal: uint64 - max_reveal_lateness: Epoch -``` - -#### `BeaconState` - -```python -class BeaconState(Container): - custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] - custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] - custody_challenge_index: uint64 - - # Future derived secrets already exposed; contains the indices of the exposed validator - # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], - EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] -``` - -#### `BeaconBlockBody` - -```python -class BeaconBlockBody(Container): - custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] - custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] - custody_responses: List[CustodyResponse, PLACEHOLDER] - custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] -``` ## Helpers @@ -732,12 +693,9 @@ def process_bit_challenge_response(state: BeaconState, ### Handling of custody-related deadlines -Run `process_reveal_deadlines(state)` immediately after `process_registry_updates(state)`: +Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: ```python -# begin insert @process_reveal_deadlines - process_reveal_deadlines(state) -# end insert @process_reveal_deadlines def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validators): deadline = validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) @@ -748,9 +706,6 @@ def process_reveal_deadlines(state: BeaconState) -> None: Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: ```python -# begin insert @process_challenge_deadlines - process_challenge_deadlines(state) -# end insert @process_challenge_deadlines def process_challenge_deadlines(state: BeaconState) -> None: for custody_chunk_challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: @@ -765,13 +720,10 @@ def process_challenge_deadlines(state: BeaconState) -> None: records[records.index(custody_bit_challenge)] = CustodyBitChallengeRecord() ``` -Append this to `process_final_updates(state)`: +After `process_final_updates(state)`, additional updates are made for the custody game: ```python -# begin insert @after_process_final_updates - after_process_final_updates(state) -# end insert @after_process_final_updates -def after_process_final_updates(state: BeaconState) -> None: +def process_custody_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] diff --git a/specs/core/1_fraudproofs.md b/specs/core/1_fraudproofs.md new file mode 100644 index 0000000000..46ba390cdd --- /dev/null +++ b/specs/core/1_fraudproofs.md @@ -0,0 +1,52 @@ +# Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + TODO + + + +## Introduction + +This document describes the shard transition function (data layer only) and the shard fork choice rule as part of Phase 1 of Ethereum 2.0. + +## Fraud proofs + +TODO. The intent is to have a single universal fraud proof type, which contains the following parts: + +1. An on-time attestation on some `shard` signing a `ShardTransition` +2. An index `i` of a particular position to focus on +3. The `ShardTransition` itself +4. The full body of the block +5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing + +The proof verifies that one of the two conditions is false: + +1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` +2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer_index(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) + +## Shard state transition function + +```python +def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: + # We will add something more substantive in phase 2 + return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) +``` + +## Honest committee member behavior + +Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `slot`, run the following procedure: + +* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. +* For `slot in get_offset_slots(state, start_slot)`, do the following: + * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. + * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` + * If `len(choices) == 1`, do `proposals.append(choices[0])` + * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. + * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. + +Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. diff --git a/specs/core/1_beacon-chain-misc.md b/specs/old/1_beacon-chain-misc.md similarity index 100% rename from specs/core/1_beacon-chain-misc.md rename to specs/old/1_beacon-chain-misc.md diff --git a/specs/core/1_shard-data-chains.md b/specs/old/1_shard-data-chains.md similarity index 100% rename from specs/core/1_shard-data-chains.md rename to specs/old/1_shard-data-chains.md From 7d2341b40d4e4353395535250b578819c8a8668d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 22:35:07 +0100 Subject: [PATCH 048/111] Note: is_valid_indexed_attestation is overriden --- specs/core/1_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 04df84e866..57bd1fcd72 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -377,7 +377,7 @@ def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: ### Predicates -#### `is_valid_indexed_attestation` +#### New `is_valid_indexed_attestation` ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: AttestationAndCommittee) -> bool: From edef2fd8ae4aa8a0a352a1ce00369d2f47a89c44 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 23:42:28 +0100 Subject: [PATCH 049/111] more refactoring: more immediate custody game, general phase1 beacon chain changes --- specs/core/1_beacon-chain.md | 130 ++++++++++++++++++------------ specs/core/1_custody-game.md | 148 +++++++++++------------------------ 2 files changed, 125 insertions(+), 153 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 57bd1fcd72..94dfd624f3 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -180,6 +180,32 @@ class Validator(Container): max_reveal_lateness: Epoch ``` +### New extended `BeaconBlockBody` + +```python +class BeaconBlockBody(phase0.BeaconBlockBody): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Slashings + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + # Attesting + attestations: List[Attestation, MAX_ATTESTATIONS] + # Enty & exit + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] + # Custody game + custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] + custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] + custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] + # Shards + shard_transitions: Vector[ShardTransition, MAX_SHARDS] + # Light clients + light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_signature: BLSSignature +``` ### New extended `BeaconBlock` @@ -189,17 +215,6 @@ class BeaconBlock(phase0.BeaconBlock): parent_root: Hash state_root: Hash body: BeaconBlockBody - shard_transitions: Vector[ShardTransition, MAX_SHARDS] - light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] - light_client_signature: BLSSignature - - # TODO: older pre-proposal custody field additions, keep this? - custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] - custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] - custody_responses: List[CustodyResponse, PLACEHOLDER] - custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] - signature: BLSSignature ``` @@ -240,10 +255,8 @@ class BeaconState(phase0.BeaconState): online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee next_light_committee: CompactCommittee - - # TODO older pre-proposal custody field additions, keep this? - custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] - custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] + + # TODO: custody game refactor, no challenge-records, immediate processing. custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS @@ -439,11 +452,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: process_operations(body.attester_slashings, process_attester_slashing) # New attestation processing - process_attestations(state, block, body.attestations) + process_attestations(state, body, body.attestations) process_operations(body.deposits, process_deposit) process_operations(body.voluntary_exits, process_voluntary_exit) + # See custody game spec. + process_custody_game_operations(state, body) + # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` @@ -526,7 +542,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr ###### `process_attestations` ```python -def process_attestations(state: BeaconState, block: BeaconBlock, attestations: Sequence[Attestation]) -> None: +def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: pending_attestations = [] # Basic validation for attestation in attestations: @@ -548,25 +564,25 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S participating_attestations.append(attestation) if attestation.data.shard_transition_root == shard_transition_root: all_participants = all_participants.union(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) - if ( - get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= - get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 - and success is False - ): - # Attestation <-> shard transition consistency - assert shard_transition_root == hash_tree_root(block.shard_transition) - assert attestation.data.head_shard_root == chunks_to_body_root(block.shard_transition.shard_data_roots[-1]) - # Apply transition - apply_shard_transition(state, shard, block.shard_transition) - # Apply proposer reward and cost - estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) - increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) - for shard_state, slot, length in zip(block.shard_transition.shard_states, offset_slots, block.shard_transition.shard_block_lengths): - decrease_balance(state, get_shard_proposer_index(state, shard, slot), shard_state.gasprice * length) - winners.add((shard, shard_transition_root)) - success = True + if ( + get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= + get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 + and success is False + ): + # Attestation <-> shard transition consistency + assert shard_transition_root == hash_tree_root(block_body.shard_transition) + assert attestation.data.head_shard_root == chunks_to_body_root(block_body.shard_transition.shard_data_roots[-1]) + # Apply transition + apply_shard_transition(state, shard, block_body.shard_transition) + # Apply proposer reward and cost + estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) + increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) + for shard_state, slot, length in zip(block_body.shard_transition.shard_states, offset_slots, block_body.shard_transition.shard_block_lengths): + decrease_balance(state, get_shard_proposer_index(state, shard, slot), shard_state.gasprice * length) + winners.add((shard, shard_transition_root)) + success = True if not success: - assert block.shard_transitions[shard] == ShardTransition() + assert block_body.shard_transitions[shard] == ShardTransition() for attestation in attestations: pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, @@ -586,22 +602,22 @@ def process_attestations(state: BeaconState, block: BeaconBlock, attestations: S #### Shard transition false positives ```python -def verify_shard_transition_false_positives(state: BeaconState, block: BeaconBlock) -> None: +def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(MAX_SHARDS): if state.shard_states[shard].slot != state.slot - 1: - assert block.shard_transition[shard] == ShardTransition() + assert block_body.shard_transition[shard] == ShardTransition() ``` #### Light client processing ```python -def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> None: +def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockBody) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) - assert len(block.light_client_signature_bitfield) == len(committee) + assert len(block_body.light_client_signature_bitfield) == len(committee) total_reward = Gwei(0) signer_keys = [] - for i, bit in enumerate(block.light_client_signature_bitfield): + for i, bit in enumerate(block_body.light_client_signature_bitfield): if bit: signer_keys.append(state.validators[committee[i]].pubkey) increase_balance(state, committee[i], get_base_reward(state, committee[i])) @@ -612,7 +628,7 @@ def process_light_client_signatures(state: BeaconState, block: BeaconBlock) -> N assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), message_hash=get_block_root_at_slot(state, state.slot - 1), - signature=block.light_client_signature, + signature=block_body.light_client_signature, domain=DOMAIN_LIGHT_CLIENT ) ``` @@ -627,26 +643,38 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) - # TODO process_reveal_deadlines - # TODO process_challenge_deadlines + process_reveal_deadlines(state) + process_challenge_deadlines(state) process_slashings(state) - # TODO update_period_committee process_final_updates(state) - # TODO process_custody_final_updates + process_custody_final_updates(state) + process_online_tracking(state) + process_light_client_committee_updates(state) +``` +#### Online-tracking + +```python +def process_online_tracking(state: BeaconState) -> None: # Slowly remove validators from the "online" set if they do not show up for index in range(len(state.validators)): if state.online_countdown[index] != 0: state.online_countdown[index] = state.online_countdown[index] - 1 - - # Update light client committees - if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: - state.current_light_committee = state.next_light_committee - new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) - state.next_light_committee = committee_to_compact_committee(state, new_committee) # Process pending attestations for pending_attestation in state.current_epoch_attestations + state.previous_epoch_attestations: for index in get_attesting_indices(state, pending_attestation.data, pending_attestation.aggregation_bits): state.online_countdown[index] = ONLINE_PERIOD ``` + +#### Light client committee updates + +```python +def process_light_client_committee_updates(state: BeaconState) -> None: + # Update light client committees + if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: + state.current_light_committee = state.next_light_committee + new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) + state.next_light_committee = committee_to_compact_committee(state, new_committee) +``` + diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 087dcdbf9a..9e880d02b1 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -17,7 +17,6 @@ - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domain types](#signature-domain-types) - - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [Custody objects](#custody-objects) - [`CustodyChunkChallenge`](#custodychunkchallenge) @@ -71,8 +70,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether - **Custody key**— - **Custody key reveal**— - **Custody key mask**— -- **Custody response**— -- **Custody response deadline**— ## Constants @@ -115,7 +112,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | | `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | | `MAX_CUSTODY_BIT_CHALLENGES` | `2**2` (= 4) | -| `MAX_CUSTODY_RESPONSES` | `2**5` (= 32) | ### Reward and penalty quotients @@ -131,11 +127,6 @@ The following types are defined, mapping into `DomainType` (little endian): | - | - | | `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | -### TODO PLACEHOLDER - -| Name | Value | -| - | - | -| `PLACEHOLDER` | `2**32` | ## Data structures @@ -189,19 +180,7 @@ class CustodyBitChallengeRecord(Container): responder_key: BLSSignature ``` -#### `CustodyResponse` - -```python -class CustodyResponse(Container): - challenge_index: uint64 - chunk_index: uint64 - chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32, CUSTODY_DATA_DEPTH] - chunk_bits_branch: List[Bytes32, CUSTODY_CHUNK_BIT_DEPTH] - chunk_bits_leaf: Bitvector[256] -``` - -### New beacon operations +### New Beacon Chain operations #### `CustodyKeyReveal` @@ -363,16 +342,27 @@ def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int ## Per-block processing -### Operations +### Custody Game Operations -Add the following operations to the per-block processing, in the order given below and after all other operations in Phase 0. +```python +def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) -> None: + assert len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS + assert len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS + assert len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGES + assert len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALLENGES + + def process_operations(operations, fn): + for operation in operations: + fn(state, operation) + + process_operations(body.custody_key_reveals, process_custody_key_reveal) + process_operations(body.early_derived_secret_reveals, process_early_derived_secret_reveal) + process_operations(body.custody_chunk_challenges, process_chunk_challenge) + process_operations(body.custody_bit_challenges, process_bit_challenge) +``` #### Custody key reveals -Verify that `len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS`. - -For each `reveal` in `block.body.custody_key_reveals`, run the following function: - ```python def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> None: """ @@ -425,10 +415,6 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> #### Early derived secret reveals -Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS`. - -For each `reveal` in `block.body.early_derived_secret_reveals`, run the following function: - ```python def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerivedSecretReveal) -> None: """ @@ -499,10 +485,6 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived #### Chunk challenges -Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALLENGES`. - -For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: - ```python def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation @@ -541,11 +523,35 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` -#### Bit challenges +TODO: immediate challenge processing, no state records. -Verify that `len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGES`. +```python +def process_chunk_challenge_response(state: BeaconState, + response: CustodyResponse, + challenge: CustodyChunkChallengeRecord) -> None: + # Verify chunk index + assert response.chunk_index == challenge.chunk_index + # Verify bit challenge data is null + assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == Hash() + # Verify minimum delay + assert get_current_epoch(state) >= challenge.inclusion_epoch + MAX_SEED_LOOKAHEAD + # Verify the chunk matches the crosslink data root + assert is_valid_merkle_branch( + leaf=hash_tree_root(response.chunk), + branch=response.data_branch, + depth=challenge.depth, + index=response.chunk_index, + root=challenge.data_root, + ) + # Clear the challenge + records = state.custody_chunk_challenge_records + records[records.index(challenge)] = CustodyChunkChallengeRecord() + # Reward the proposer + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) +``` -For each `challenge` in `block.body.custody_bit_challenges`, run the following function: +#### Bit challenges ```python def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: @@ -606,52 +612,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` -#### Custody responses - -Verify that `len(block.body.custody_responses) <= MAX_CUSTODY_RESPONSES`. - -For each `response` in `block.body.custody_responses`, run the following function: - -```python -def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: - chunk_challenge = next((record for record in state.custody_chunk_challenge_records - if record.challenge_index == response.challenge_index), None) - if chunk_challenge is not None: - return process_chunk_challenge_response(state, response, chunk_challenge) - - bit_challenge = next((record for record in state.custody_bit_challenge_records - if record.challenge_index == response.challenge_index), None) - if bit_challenge is not None: - return process_bit_challenge_response(state, response, bit_challenge) - - assert False -``` - -```python -def process_chunk_challenge_response(state: BeaconState, - response: CustodyResponse, - challenge: CustodyChunkChallengeRecord) -> None: - # Verify chunk index - assert response.chunk_index == challenge.chunk_index - # Verify bit challenge data is null - assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == Bytes32() - # Verify minimum delay - assert get_current_epoch(state) >= challenge.inclusion_epoch + MAX_SEED_LOOKAHEAD - # Verify the chunk matches the crosslink data root - assert is_valid_merkle_branch( - leaf=hash_tree_root(response.chunk), - branch=response.data_branch, - depth=challenge.depth, - index=response.chunk_index, - root=challenge.data_root, - ) - # Clear the challenge - records = state.custody_chunk_challenge_records - records[records.index(challenge)] = CustodyChunkChallengeRecord() - # Reward the proposer - proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) -``` +TODO: immediate challenge processing, no state records. ```python def process_bit_challenge_response(state: BeaconState, @@ -703,23 +664,6 @@ def process_reveal_deadlines(state: BeaconState) -> None: slash_validator(state, ValidatorIndex(index)) ``` -Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: - -```python -def process_challenge_deadlines(state: BeaconState) -> None: - for custody_chunk_challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: - slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) - records = state.custody_chunk_challenge - records[records.index(custody_chunk_challenge)] = CustodyChunkChallengeRecord() - - for custody_bit_challenge in state.custody_bit_challenge_records: - if get_current_epoch(state) > custody_bit_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: - slash_validator(state, custody_bit_challenge.responder_index, custody_bit_challenge.challenger_index) - records = state.custody_bit_challenge_records - records[records.index(custody_bit_challenge)] = CustodyBitChallengeRecord() -``` - After `process_final_updates(state)`, additional updates are made for the custody game: ```python From 465f6cb7f020c969ca83f2f11c604841225ada6e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 23:46:31 +0100 Subject: [PATCH 050/111] update readme spec links --- README.md | 4 ++-- specs/core/{1_fraudproofs.md => 1_fraud_proofs.md} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename specs/core/{1_fraudproofs.md => 1_fraud_proofs.md} (94%) diff --git a/README.md b/README.md index fa103394d0..1bb2ae49fc 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Core specifications for Eth2 client validation can be found in [specs/core](spec * [Honest Validator](specs/validator/0_beacon-chain-validator.md) ### Phase 1 +* [The Beacon Chain for Shards](specs/core/1_beacon-chain.md) * [Custody Game](specs/core/1_custody-game.md) -* [Shard Data Chains](specs/core/1_shard-data-chains.md) -* [Misc beacon chain updates](specs/core/1_beacon-chain-misc.md) +* [Shard Transition and Fraud Proofs](specs/core/1_fraud_proofs.md) ### Phase 2 diff --git a/specs/core/1_fraudproofs.md b/specs/core/1_fraud_proofs.md similarity index 94% rename from specs/core/1_fraudproofs.md rename to specs/core/1_fraud_proofs.md index 46ba390cdd..e44f790eda 100644 --- a/specs/core/1_fraudproofs.md +++ b/specs/core/1_fraud_proofs.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Phase 1 -- Crosslinks and Shard Data +# Ethereum 2.0 Phase 1 -- Shard Transition and Fraud Proofs **Notice**: This document is a work-in-progress for researchers and implementers. @@ -12,7 +12,7 @@ ## Introduction -This document describes the shard transition function (data layer only) and the shard fork choice rule as part of Phase 1 of Ethereum 2.0. +This document describes the shard transition function and fraud proofs as part of Phase 1 of Ethereum 2.0. ## Fraud proofs From 589d5a4f9ae06fe6b49acfdfab46e13024b9f7db Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 23:52:57 +0100 Subject: [PATCH 051/111] Add notes on custody game spec funcs --- specs/core/1_beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 94dfd624f3..169c711238 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -644,7 +644,6 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_reveal_deadlines(state) - process_challenge_deadlines(state) process_slashings(state) process_final_updates(state) process_custody_final_updates(state) @@ -652,6 +651,10 @@ def process_epoch(state: BeaconState) -> None: process_light_client_committee_updates(state) ``` +#### Custody game updates + +`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./1_custody-game.md), + #### Online-tracking ```python From 74d6021507dc5b54cbfde25a9bc5b280a94b44cd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Nov 2019 11:13:47 +0100 Subject: [PATCH 052/111] PR comment fixes --- specs/core/0_beacon-chain.md | 14 +++++++----- specs/core/1_beacon-chain.md | 43 +++++++++++++++++++----------------- specs/core/1_custody-game.md | 15 +++++-------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e8aa53f13b..3b204e2c74 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1419,11 +1419,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - process_operations(body.proposer_slashings, process_proposer_slashing) - process_operations(body.attester_slashings, process_attester_slashing) - process_operations(body.attestations, process_attestations) - process_operations(body.deposits, process_deposit) - process_operations(body.voluntary_exits, process_voluntary_exit) + def for_ops(operations, fn): + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestations) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) ``` ##### Proposer slashings diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 169c711238..4e98445f2b 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -146,10 +146,10 @@ class AttestationCustodyBitWrapper(Container): bit: bool ``` -### New `PendingAttestation` +### New extended `PendingAttestation` ```python -class PendingAttestation(Container): +class PendingAttestation(phase0.PendingAttestation): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData inclusion_delay: Slot @@ -160,7 +160,7 @@ class PendingAttestation(Container): ### New extended `Validator` ```python -class Validator(Container): +class Validator(phase0.Validator): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals effective_balance: Gwei # Balance at stake @@ -170,9 +170,7 @@ class Validator(Container): activation_epoch: Epoch exit_epoch: Epoch withdrawable_epoch: Epoch # When validator can withdraw funds - - # TODO: older pre-proposal custody field additions, keep this? - # + # Custody game # next_custody_secret_to_reveal is initialised to the custody period # (of the particular validator) in which the validator is activated # = get_custody_period_for_validator(...) @@ -196,10 +194,10 @@ class BeaconBlockBody(phase0.BeaconBlockBody): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game - custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] - custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] - custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] + custody_chunk_challenges: List[CustodyChunkChallenge, MAX_CUSTODY_CHUNK_CHALLENGES] + custody_bit_challenges: List[CustodyBitChallenge, MAX_CUSTODY_BIT_CHALLENGES] + custody_key_reveals: List[CustodyKeyReveal, MAX_CUSTODY_KEY_REVEALS] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, MAX_EARLY_DERIVED_SECRET_REVEALS] # Shards shard_transitions: Vector[ShardTransition, MAX_SHARDS] # Light clients @@ -209,6 +207,8 @@ class BeaconBlockBody(phase0.BeaconBlockBody): ### New extended `BeaconBlock` +Note that the `body` has a new `BeaconBlockBody` definition. + ```python class BeaconBlock(phase0.BeaconBlock): slot: Slot @@ -220,6 +220,8 @@ class BeaconBlock(phase0.BeaconBlock): ### New extended `BeaconState` +Note that aside from the new additions, `Validator` and `PendingAttestation` have new definitions. + ```python class BeaconState(phase0.BeaconState): # Versioning @@ -312,10 +314,10 @@ def chunks_to_body_root(chunks): ### Beacon state accessors -#### `get_online_validators` +#### `get_online_validator_indices` ```python -def get_online_indices(state: BeaconState) -> Set[ValidatorIndex]: +def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: active_validators = get_active_validator_indices(state, get_current_epoch(state)) return set([i for i in active_validators if state.online_countdown[i] != 0]) ``` @@ -390,7 +392,9 @@ def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: ### Predicates -#### New `is_valid_indexed_attestation` +#### Updated `is_valid_indexed_attestation` + +Note that this replaces the Phase 0 `is_valid_indexed_attestation`. ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: AttestationAndCommittee) -> bool: @@ -444,18 +448,18 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - def process_operations(operations, fn): + def for_ops(operations, fn): for operation in operations: fn(state, operation) - process_operations(body.proposer_slashings, process_proposer_slashing) - process_operations(body.attester_slashings, process_attester_slashing) + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) # New attestation processing process_attestations(state, body, body.attestations) - process_operations(body.deposits, process_deposit) - process_operations(body.voluntary_exits, process_voluntary_exit) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) # See custody game spec. process_custody_game_operations(state, body) @@ -543,12 +547,11 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr ```python def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: - pending_attestations = [] # Basic validation for attestation in attestations: validate_attestation(state, attestation) # Process crosslinks - online_indices = get_online_indices(state) + online_indices = get_online_validator_indices(state) winners = set() for shard in range(ACTIVE_SHARDS): success = False diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9e880d02b1..0e804f628c 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -346,19 +346,14 @@ def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int ```python def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) -> None: - assert len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS - assert len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS - assert len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGES - assert len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALLENGES - - def process_operations(operations, fn): + def for_ops(operations, fn): for operation in operations: fn(state, operation) - process_operations(body.custody_key_reveals, process_custody_key_reveal) - process_operations(body.early_derived_secret_reveals, process_early_derived_secret_reveal) - process_operations(body.custody_chunk_challenges, process_chunk_challenge) - process_operations(body.custody_bit_challenges, process_bit_challenge) + for_ops(body.custody_key_reveals, process_custody_key_reveal) + for_ops(body.early_derived_secret_reveals, process_early_derived_secret_reveal) + for_ops(body.custody_chunk_challenges, process_chunk_challenge) + for_ops(body.custody_bit_challenges, process_bit_challenge) ``` #### Custody key reveals From cc0aac3aaed2b85ab45e0303efc04f33ba7b1159 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Nov 2019 11:17:39 +0100 Subject: [PATCH 053/111] typo --- specs/core/1_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 4e98445f2b..e7f53865da 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -190,7 +190,7 @@ class BeaconBlockBody(phase0.BeaconBlockBody): attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] # Attesting attestations: List[Attestation, MAX_ATTESTATIONS] - # Enty & exit + # Entry & exit deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game From 36503f98e43b24cdc82715793849fcf9e71c0c85 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Nov 2019 12:23:45 +0100 Subject: [PATCH 054/111] placeholder: rudimentary phase 0 to phase 1 fork process --- specs/core/1_beacon-chain.md | 6 +- specs/core/1_phase1_fork.md | 107 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 specs/core/1_phase1_fork.md diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index e7f53865da..74ad4f5f9c 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -253,11 +253,11 @@ class BeaconState(phase0.BeaconState): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 - shard_states: Vector[ShardState, MAX_SHARDS] + shard_states: List[ShardState, MAX_SHARDS] online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee next_light_committee: CompactCommittee - + # Custody game # TODO: custody game refactor, no challenge-records, immediate processing. custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator @@ -607,7 +607,7 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest ```python def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it - for shard in range(MAX_SHARDS): + for shard in range(ACTIVE_SHARDS): if state.shard_states[shard].slot != state.slot - 1: assert block_body.shard_transition[shard] == ShardTransition() ``` diff --git a/specs/core/1_phase1_fork.md b/specs/core/1_phase1_fork.md new file mode 100644 index 0000000000..f5ab658f5c --- /dev/null +++ b/specs/core/1_phase1_fork.md @@ -0,0 +1,107 @@ +# Ethereum 2.0 Phase 1 -- From Phase 0 to Phase 1 + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + TODO + + + +## Introduction + +This document describes the process of moving from Phase 0 to Phase 1 of Ethereum 2.0. + +## Configuration + +TODO: very unstable/experimental. PLACEHOLDER. + +| Name | Value | Unit | +| - | - | - | +| `PHASE_1_FORK_VERSION` | `0x00000001` | `Version` | +| `INITIAL_GASPRICE` | `10` | `Gwei` | + +## Fork to Phase 1 + +### Fork trigger. + +TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. + +### Upgrading the state + +After `process_slots` of Phase 0 finishes, but before the first Phase 1 block is processed, an irregular state change is made to upgrade to Phase 1. + +```python +def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: + post = BeaconState( + genesis_time=pre.genesis_time, + slot=pre.slot, + fork=Fork( + previous_version=pre.current_version, + current_version=PHASE_1_FORK_VERSION, + epoch=get_current_epoch(pre), + ), + # History + latest_block_header=pre.latest_block_header, + block_roots=pre.block_roots, + state_roots=pre.state_roots, + historical_roots=pre.historical_roots, + # Eth1 + eth1_data=pre.eth1_data, + eth1_data_votes=pre.eth1_data_votes, + eth1_deposit_index=pre.eth1_deposit_index, + # Registry + validators=List[Validator, VALIDATOR_REGISTRY_LIMIT]( + Validator( + pubkey=phase0_validator.pubkey, + withdrawal_credentials=phase0_validator.withdrawal_credentials, + effective_balance=phase0_validator.effective_balance, + slashed=phase0.slashed, + activation_eligibility_epoch=phase0_validator.activation_eligibility_epoch, + activation_epoch=phase0_validator.activation_eligibility_epoch, + exit_epoch=phase0_validator.exit_epoch, + withdrawable_epoch=phase0_validator.withdrawable_epoch, + next_custody_secret_to_reveal=, + max_reveal_lateness=, + ) for phase0_validator in pre.validators + ), + balances=pre.balances, + # Randomness + randao_mixes=pre.randao_mixes, + # Slashings + slashings=pre.slashings, + # Attestations + # previous_epoch_attestations is cleared on upgrade. + previous_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + # empty in pre state, since the upgrade is performed just after an epoch boundary. + current_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + # Finality + justification_bits=pre.justification_bits, + previous_justified_checkpoint=pre.previous_justified_checkpoint, + current_justified_checkpoint=pre.current_justified_checkpoint, + finalized_checkpoint=pre.finalized_checkpoint, + # Phase 1 + shard_states=List[ShardState, MAX_SHARDS]( + ShardState( + slot=0, + gasprice=INITIAL_GASPRICE, + data=Root(), + latest_block_root=Hash(), + ) for i in range(ACTIVE_SHARDS) + ), + online_countdown=Bytes[VALIDATOR_REGISTRY_LIMIT]( + ONLINE_PERIOD for i in range(len(pre.validators)) + ), + current_light_committee=CompactCommittee(), # computed after state creation + next_light_committee=CompactCommittee(), + # Custody game + custody_challenge_index=0, + exposed_derived_secrets=Vector[List[ValidatorIndex, PLACEHOLDER], + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() + ) + post.current_light_committee = get_light_client_committee(post, post.epoch) + post.next_light_committee = get_light_client_committee(post, post.epoch + 1) + return post +``` From 4c9e5b89501ee84e66404100275fb6dadbaf77c7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Nov 2019 12:25:13 +0100 Subject: [PATCH 055/111] add link to phase1 upgrade doc --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1bb2ae49fc..b801cbd3d1 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Core specifications for Eth2 client validation can be found in [specs/core](spec * [Honest Validator](specs/validator/0_beacon-chain-validator.md) ### Phase 1 +* [From Phase 0 to Phase 1](specs/core/1_phase1_fork.md) * [The Beacon Chain for Shards](specs/core/1_beacon-chain.md) * [Custody Game](specs/core/1_custody-game.md) * [Shard Transition and Fraud Proofs](specs/core/1_fraud_proofs.md) From 03e956f9c95fc59094a9d199e90e5796dabab650 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Nov 2019 12:33:24 +0100 Subject: [PATCH 056/111] validator init adjustments --- specs/core/1_custody-game.md | 10 +++++----- specs/core/1_phase1_fork.md | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0e804f628c..e4e7527633 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -320,11 +320,11 @@ def get_randao_epoch_for_custody_period(period: uint64, validator_index: Validat ### `get_custody_period_for_validator` ```python -def get_custody_period_for_validator(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: +def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epoch=None) -> int: ''' Return the reveal period for a given validator. ''' - epoch = get_current_epoch(state) if epoch is None else epoch + epoch = if epoch is None else epoch return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD ``` @@ -367,7 +367,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_secret_to_reveal, reveal.revealer_index) - assert revealer.next_custody_secret_to_reveal < get_custody_period_for_validator(state, reveal.revealer_index) + assert revealer.next_custody_secret_to_reveal < get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -566,7 +566,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] assert get_current_epoch(state) <= get_randao_epoch_for_custody_period( - get_custody_period_for_validator(state, challenge.responder_index, epoch), + get_custody_period_for_validator(challenge.responder_index, epoch), challenge.responder_index ) + 2 * EPOCHS_PER_CUSTODY_PERIOD + responder.max_reveal_lateness @@ -578,7 +578,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> assert record.challenger_index != challenge.challenger_index # Verify the responder custody key epoch_to_sign = get_randao_epoch_for_custody_period( - get_custody_period_for_validator(state, challenge.responder_index, epoch), + get_custody_period_for_validator(challenge.responder_index, epoch), challenge.responder_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) diff --git a/specs/core/1_phase1_fork.md b/specs/core/1_phase1_fork.md index f5ab658f5c..e257d6eb1d 100644 --- a/specs/core/1_phase1_fork.md +++ b/specs/core/1_phase1_fork.md @@ -35,13 +35,14 @@ After `process_slots` of Phase 0 finishes, but before the first Phase 1 block is ```python def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: + epoch = get_current_epoch(pre) post = BeaconState( genesis_time=pre.genesis_time, slot=pre.slot, fork=Fork( previous_version=pre.current_version, current_version=PHASE_1_FORK_VERSION, - epoch=get_current_epoch(pre), + epoch=epoch, ), # History latest_block_header=pre.latest_block_header, @@ -58,14 +59,14 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: pubkey=phase0_validator.pubkey, withdrawal_credentials=phase0_validator.withdrawal_credentials, effective_balance=phase0_validator.effective_balance, - slashed=phase0.slashed, + slashed=phase0_validator.slashed, activation_eligibility_epoch=phase0_validator.activation_eligibility_epoch, activation_epoch=phase0_validator.activation_eligibility_epoch, exit_epoch=phase0_validator.exit_epoch, withdrawable_epoch=phase0_validator.withdrawable_epoch, - next_custody_secret_to_reveal=, - max_reveal_lateness=, - ) for phase0_validator in pre.validators + next_custody_secret_to_reveal=get_custody_period_for_validator(validator_index, epoch), + max_reveal_lateness=0, # TODO custody refactor. Outdated? + ) for validator_index, phase0_validator in enumerate(pre.validators) ), balances=pre.balances, # Randomness From 79be0345d61a7d6ec96f7f45d1a9369f6bce2aee Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Nov 2019 13:47:53 -0700 Subject: [PATCH 057/111] fix call to process_attestation --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3b204e2c74..32c10c4ecd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1425,7 +1425,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.proposer_slashings, process_proposer_slashing) for_ops(body.attester_slashings, process_attester_slashing) - for_ops(body.attestations, process_attestations) + for_ops(body.attestations, process_attestation) for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) ``` From 82c9adce8baea5ef1961720629979b2fc9381c4e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Nov 2019 14:04:29 -0700 Subject: [PATCH 058/111] fix phase 1 make build (not lint or tests) --- Makefile | 2 +- scripts/build_spec.py | 21 +++++++++++-------- specs/core/1_beacon-chain.md | 4 ++-- .../{1_fraud_proofs.md => 1_fraud-proofs.md} | 0 .../{1_phase1_fork.md => 1_phase1-fork.md} | 0 5 files changed, 15 insertions(+), 12 deletions(-) rename specs/core/{1_fraud_proofs.md => 1_fraud-proofs.md} (100%) rename specs/core/{1_phase1_fork.md => 1_phase1-fork.md} (100%) diff --git a/Makefile b/Makefile index bfbc280701..d88c91651e 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_beacon-chain.md $(SPEC_DIR)/core/1_fraud-proofs.md $(SPEC_DIR)/core/1_phase1-fork.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 9fb52eb1c5..8bc4398cc7 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -289,16 +289,18 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, phase0_fork_choice_sourcefile: str, merkle_proofs_sourcefile: str, phase1_custody_sourcefile: str, - phase1_shard_sourcefile: str, - phase1_beacon_misc_sourcefile: str, + phase1_beacon_sourcefile: str, + phase1_fraud_sourcefile: str, + phase1_fork_sourcefile: str, outfile: str=None) -> Optional[str]: all_sourcefiles = ( phase0_beacon_sourcefile, phase0_fork_choice_sourcefile, merkle_proofs_sourcefile, phase1_custody_sourcefile, - phase1_shard_sourcefile, - phase1_beacon_misc_sourcefile, + phase1_beacon_sourcefile, + phase1_fraud_sourcefile, + phase1_fork_sourcefile, ) all_spescs = [get_spec(spec) for spec in all_sourcefiles] for spec in all_spescs: @@ -327,9 +329,10 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, 2nd argument is input /core/0_fork-choice.md 3rd argument is input /light_client/merkle_proofs.md 4th argument is input /core/1_custody-game.md - 5th argument is input /core/1_shard-data-chains.md - 6th argument is input /core/1_beacon-chain-misc.md - 7th argument is output spec.py + 5th argument is input /core/1_beacon-chain.md + 6th argument is input /core/1_fraud-proofs.md + 7th argument is input /core/1_phase1-fork.md + 8th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -342,14 +345,14 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, else: print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: - if len(args.files) == 7: + if len(args.files) == 8: build_phase1_spec(*args.files) else: print( " Phase 1 requires input files as well as an output file:\n" "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n" "\t light_client: (merkle_proofs.md)\n" - "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n" + "\t core/phase_1: (1_custody-game.md, 1_beacon-chain.md, 1_fraud-proofs.md, 1_phase1-fork.md)\n" "\t and output.py" ) else: diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 74ad4f5f9c..1cb1c35252 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -143,7 +143,7 @@ class CompactCommittee(Container): class AttestationCustodyBitWrapper(Container): attestation_root: Hash block_index: uint64 - bit: bool + bit: boolean ``` ### New extended `PendingAttestation` @@ -154,7 +154,7 @@ class PendingAttestation(phase0.PendingAttestation): data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex - crosslink_success: bool + crosslink_success: boolean ``` ### New extended `Validator` diff --git a/specs/core/1_fraud_proofs.md b/specs/core/1_fraud-proofs.md similarity index 100% rename from specs/core/1_fraud_proofs.md rename to specs/core/1_fraud-proofs.md diff --git a/specs/core/1_phase1_fork.md b/specs/core/1_phase1-fork.md similarity index 100% rename from specs/core/1_phase1_fork.md rename to specs/core/1_phase1-fork.md From 9f80a7f703dce4bad748875a212949f3653d1689 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Nov 2019 15:07:50 -0700 Subject: [PATCH 059/111] phase1 lint (1) --- scripts/build_spec.py | 8 +++--- specs/core/1_beacon-chain.md | 50 ++++++++++++++++++++++++++---------- specs/core/1_custody-game.md | 6 ++--- specs/core/1_fraud-proofs.md | 7 ++++- specs/core/1_phase1-fork.md | 4 +-- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 8bc4398cc7..d6a06cbbcc 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -171,7 +171,7 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' - + '\n\n '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n ', value[:-1])) + + '\n\n '.join([strip_comments(re.sub(r'\n\n', r'\n', re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]))) for value in ssz_objects.values()]) + '\n\n' + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) @@ -243,10 +243,8 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str] """ for key, value in new_objects.items(): if key in old_objects: - # remove trailing newline - old_objects[key] = old_objects[key] - # remove leading variable name - value = re.sub(r'^class [\w]*\(Container\):\n', '', value) + # add proper spacing + old_objects[key] = old_objects[key] + "\n\n" old_objects[key] = old_objects.get(key, '') + value dependency_order_ssz_objects(old_objects, custom_types) return old_objects diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 1cb1c35252..708937059a 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -81,7 +81,7 @@ class ShardState(Container): ### New `AttestationData` ```python -class AttestationData(Container): +class AttestationData(phase0.AttestationData): slot: Slot index: CommitteeIndex # LMD GHOST vote @@ -417,7 +417,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Attest AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, cbit) )) else: - assert cbit == False + assert not cbit return bls_verify_multiple( pubkeys=all_pubkeys, @@ -503,7 +503,12 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Correct data root count offset_slots = get_offset_slots(state, start_slot) - assert len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) + assert ( + len(transition.shard_data_roots) + == len(transition.shard_states) + == len(transition.shard_block_lengths) + == len(offset_slots) + ) assert transition.start_slot == start_slot # Reonstruct shard headers @@ -514,7 +519,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr if any(transition.shard_data_roots): headers.append(ShardSignableHeader( shard_parent_root=shard_parent_root, - parent_hash=get_block_root_at_slot(state, state.slot-1), + parent_hash=get_block_root_at_slot(state, state.slot - 1), slot=offset_slots[i], body_root=chunks_to_body_root(transition.shard_data_roots[i]) )) @@ -524,8 +529,10 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Verify correct calculation of gas prices and slots and chunk roots prev_gasprice = state.shard_states[shard].gasprice for i in range(len(offset_slots)): - shard_state, block_length, chunks = transition.shard_states[i], transition.shard_block_lengths[i], transition.shard_data_roots[i] - assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) + shard_state = transition.shard_states[i] + block_length = transition.shard_block_lengths[i] + chunks = transition.shard_data_roots[i] + assert dfhard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) assert shard_state.slot == offset_slots[i] assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE prev_gasprice = shard_state.gasprice @@ -549,24 +556,29 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: # Basic validation for attestation in attestations: - validate_attestation(state, attestation) + validate_attestation(state, attestation) # Process crosslinks online_indices = get_online_validator_indices(state) winners = set() for shard in range(ACTIVE_SHARDS): success = False # All attestations in the block for this shard - this_shard_attestations = [attestation for attestation in attestations if get_shard(state, attestation) == shard and attestation.data.slot == state.slot] + this_shard_attestations = [ + attestation for attestation in attestations + if get_shard(state, attestation) == shard and attestation.data.slot == state.slot + ] # The committee for this shard this_shard_committee = get_beacon_committee(state, get_current_epoch(state), shard) # Loop over all shard transition roots - for shard_transition_root in sorted(set([attestation.data.shard_transition_root for attestation in this_shard_attestations])): + shard_transition_roots = set([a.data.shard_transition_root for a in this_shard_attestations]) + for shard_transition_root in sorted(shard_transition_roots): all_participants = set() participating_attestations = [] for attestation in this_shard_attestations: participating_attestations.append(attestation) if attestation.data.shard_transition_root == shard_transition_root: - all_participants = all_participants.union(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) + participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + all_participants = all_participants.union(participants) if ( get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 @@ -574,24 +586,34 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest ): # Attestation <-> shard transition consistency assert shard_transition_root == hash_tree_root(block_body.shard_transition) - assert attestation.data.head_shard_root == chunks_to_body_root(block_body.shard_transition.shard_data_roots[-1]) + assert ( + attestation.data.head_shard_root + == chunks_to_body_root(block_body.shard_transition.shard_data_roots[-1]) + ) # Apply transition apply_shard_transition(state, shard, block_body.shard_transition) # Apply proposer reward and cost estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) - for shard_state, slot, length in zip(block_body.shard_transition.shard_states, offset_slots, block_body.shard_transition.shard_block_lengths): - decrease_balance(state, get_shard_proposer_index(state, shard, slot), shard_state.gasprice * length) + states_slots_lengths = zip( + block_body.shard_transition.shard_states, + offset_slots, + block_body.shard_transition.shard_block_lengths + ) + for shard_state, slot, length in states_slots_lengths: + proposer_index = get_shard_proposer_index(state, shard, slot) + decrease_balance(state, proposer_index, shard_state.gasprice * length) winners.add((shard, shard_transition_root)) success = True if not success: assert block_body.shard_transitions[shard] == ShardTransition() for attestation in attestations: + is_winning_transition = (get_shard(state, attestation), attestation.shard_transition_root) in winners pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, data=attestation.data, inclusion_delay=state.slot - data.slot, - crosslink_success=(get_shard(state, attestation), attestation.shard_transition_root) in winners and attestation.data.slot == state.slot, + crosslink_success=is_winning_transition and attestation.data.slot == state.slot, proposer_index=proposer_index ) if attestation.data.target.epoch == get_current_epoch(state): diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e4e7527633..689882551f 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -320,11 +320,10 @@ def get_randao_epoch_for_custody_period(period: uint64, validator_index: Validat ### `get_custody_period_for_validator` ```python -def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epoch=None) -> int: +def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epoch) -> int: ''' Return the reveal period for a given validator. ''' - epoch = if epoch is None else epoch return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD ``` @@ -367,7 +366,8 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_secret_to_reveal, reveal.revealer_index) - assert revealer.next_custody_secret_to_reveal < get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) + custody_reveal_period = get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) + assert revealer.next_custody_secret_to_reveal < custody_reveal_period # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) diff --git a/specs/core/1_fraud-proofs.md b/specs/core/1_fraud-proofs.md index e44f790eda..bd5cce117b 100644 --- a/specs/core/1_fraud-proofs.md +++ b/specs/core/1_fraud-proofs.md @@ -32,7 +32,12 @@ The proof verifies that one of the two conditions is false: ## Shard state transition function ```python -def shard_state_transition(shard: Shard, slot: Slot, pre_state: Hash, previous_beacon_root: Hash, proposer_pubkey: BLSPubkey, block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: +def shard_state_transition(shard: Shard, + slot: Slot, + pre_state: Hash, + previous_beacon_root: Hash, + proposer_pubkey: BLSPubkey, + block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: # We will add something more substantive in phase 2 return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) ``` diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index e257d6eb1d..62c8966691 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -93,14 +93,14 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: ) for i in range(ACTIVE_SHARDS) ), online_countdown=Bytes[VALIDATOR_REGISTRY_LIMIT]( - ONLINE_PERIOD for i in range(len(pre.validators)) + ONLINE_PERIOD for i in range(len(pre.validators)) ), current_light_committee=CompactCommittee(), # computed after state creation next_light_committee=CompactCommittee(), # Custody game custody_challenge_index=0, exposed_derived_secrets=Vector[List[ValidatorIndex, PLACEHOLDER], - EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() ) post.current_light_committee = get_light_client_committee(post, post.epoch) post.next_light_committee = get_light_client_committee(post, post.epoch + 1) From d6bfe5d35c498c526209c71d554b0614e06b2d92 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Nov 2019 16:40:02 -0700 Subject: [PATCH 060/111] lint phase 1 (2) --- scripts/build_spec.py | 4 +++- specs/core/1_beacon-chain.md | 39 ++++++++++++++++++++++++++---------- specs/core/1_custody-game.md | 27 +++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d6a06cbbcc..bcc6595c65 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -53,7 +53,7 @@ BasicValue, Elements, BaseBytes, BaseList, SSZType, Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector, Bits, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, - uint64, bit, boolean, byte, + uint64, uint8, bit, boolean, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -245,6 +245,8 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str] if key in old_objects: # add proper spacing old_objects[key] = old_objects[key] + "\n\n" + lines = value.split("\n") + value = "\n".join([lines[0] + " # noqa: F811"] + lines[1:]) old_objects[key] = old_objects.get(key, '') + value dependency_order_ssz_objects(old_objects, custom_types) return old_objects diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 708937059a..63c2071a79 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -15,6 +15,14 @@ TODO This document describes the extensions made to the Phase 0 design of The Beacon Chain to facilitate the new shards as part of Phase 1 of Eth2. +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `Shard` | `uint64` | a shard number | + ## Configuration Configuration is not namespaced. Instead it is strictly an extension; @@ -39,7 +47,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_LIGHT_CLIENT` | `192` | | +| `DOMAIN_LIGHT_CLIENT` | `192` | | | `DOMAIN_SHARD_COMMITTEE` | `192` | | | `DOMAIN_SHARD_PROPOSAL` | `193` | | @@ -338,8 +346,8 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, slot_to_epoch(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(state), DOMAIN_SHARD_COMMITTEE)[:8]) + committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) + r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` @@ -351,7 +359,7 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque if source_epoch > 0: source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) - seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_LIGHT_CLIENT) + seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS)[:TARGET_COMMITTEE_SIZE] ``` @@ -375,6 +383,14 @@ def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` +#### `get_start_shard` + +```python +def get_start_shard(state: BeaconState, slot: Slot) -> Shard: + # TODO: implement start shard logic + return Shard(0) +``` + #### `get_shard` ```python @@ -476,7 +492,6 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data assert data.index < ACTIVE_SHARDS shard = get_shard(state, attestation) - proposer_index = get_beacon_proposer_index(state) # Signature check assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) @@ -490,7 +505,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) # Type 2: delayed attestations else: - assert state.slot - compute_start_slot_at_epoch(slot_to_epoch(data.slot)) < EPOCH_LENGTH + assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH assert data.shard_transition_root == Hash() ``` @@ -532,7 +547,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr shard_state = transition.shard_states[i] block_length = transition.shard_block_lengths[i] chunks = transition.shard_data_roots[i] - assert dfhard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) + assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) assert shard_state.slot == offset_slots[i] assert len(chunks) == block_length // SHARD_BLOCK_CHUNK_SIZE prev_gasprice = shard_state.gasprice @@ -593,11 +608,13 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest # Apply transition apply_shard_transition(state, shard, block_body.shard_transition) # Apply proposer reward and cost + beacon_proposer_index = get_beacon_proposer_index(state) estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) - increase_balance(state, proposer, estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT) + proposer_reward = estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT + increase_balance(state, beacon_proposer_index, proposer_reward) states_slots_lengths = zip( block_body.shard_transition.shard_states, - offset_slots, + get_offset_slots(state, state.shard_next_slots[get_shard(attestation)]), block_body.shard_transition.shard_block_lengths ) for shard_state, slot, length in states_slots_lengths: @@ -612,7 +629,7 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, data=attestation.data, - inclusion_delay=state.slot - data.slot, + inclusion_delay=state.slot - attestation.data.slot, crosslink_success=is_winning_transition and attestation.data.slot == state.slot, proposer_index=proposer_index ) @@ -654,7 +671,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB pubkey=bls_aggregate_pubkeys(signer_keys), message_hash=get_block_root_at_slot(state, state.slot - 1), signature=block_body.light_client_signature, - domain=DOMAIN_LIGHT_CLIENT + domain=DOMAIN_LIGHT_CLIENT, ) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 689882551f..6c49ee4701 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -132,6 +132,21 @@ The following types are defined, mapping into `DomainType` (little endian): ### Custody objects +#### `Crosslink` + +*Note*: Crosslinks have been removed in the phase 1 redesign. This is a placeholder until the custody game is revamped. + +```python +class Crosslink(Container): + shard: uint64 + parent_root: Hash + # Crosslinking data + start_epoch: Epoch + end_epoch: Epoch + data_root: Hash +``` + + #### `CustodyChunkChallenge` ```python @@ -180,6 +195,18 @@ class CustodyBitChallengeRecord(Container): responder_key: BLSSignature ``` +#### `CustodyResponse` + +```python +class CustodyResponse(Container): + challenge_index: uint64 + chunk_index: uint64 + chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] + data_branch: List[Bytes32, PLACEHOLDER] + chunk_bits_branch: List[Bytes32, PLACEHOLDER] + chunk_bits_leaf: Bytes32 +``` + ### New Beacon Chain operations #### `CustodyKeyReveal` From e73316c13fa034ef57d67a639197a6ff6d316670 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 Nov 2019 20:16:40 +0100 Subject: [PATCH 061/111] change configs to be loaded in time; reload specs module to make new config presets effective. Also fix more lint and consistency problems. --- scripts/build_spec.py | 78 ++++--------------- scripts/function_puller.py | 25 ++---- specs/core/1_beacon-chain.md | 22 +++--- specs/core/1_custody-game.md | 76 +----------------- specs/core/1_phase1-fork.md | 2 +- test_libs/pyspec/eth2spec/config/__init__.py | 0 .../pyspec/eth2spec/config/apply_config.py | 19 +++++ test_libs/pyspec/eth2spec/test/conftest.py | 8 +- test_libs/pyspec/eth2spec/test/context.py | 11 ++- 9 files changed, 65 insertions(+), 176 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/config/__init__.py create mode 100644 test_libs/pyspec/eth2spec/config/apply_config.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index bcc6595c65..3352864378 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -9,9 +9,13 @@ Optional, ) +CONFIG_LOADER = ''' +apply_constants_preset(globals()) +''' -PHASE0_IMPORTS = '''from typing import ( - Any, Dict, Set, Sequence, Tuple, Optional +PHASE0_IMPORTS = '''from eth2spec.config.apply_config import apply_constants_preset +from typing import ( + Dict, Set, Sequence, Tuple, Optional ) from dataclasses import ( @@ -33,8 +37,10 @@ from eth2spec.utils.hash_function import hash ''' -PHASE1_IMPORTS = '''from typing import ( - Any, Dict, Set, Sequence, MutableSequence, NewType, Tuple, Union, +PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.apply_config import apply_constants_preset +from typing import ( + Dict, Set, Sequence, NewType, Tuple, Union, ) from math import ( log2, @@ -101,24 +107,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore if param_hash not in committee_cache: committee_cache[param_hash] = _compute_committee(indices, seed, index, count) - return committee_cache[param_hash] - - -# Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]) -> None: - global_vars = globals() - for k, v in preset.items(): - if k.startswith('DOMAIN_'): - global_vars[k] = DomainType(v) # domain types are defined as bytes in the configs - else: - global_vars[k] = v - - # Deal with derived constants - global_vars['GENESIS_EPOCH'] = compute_epoch_at_slot(GENESIS_SLOT) - - # Initialize SSZ types again, to account for changed lengths - init_SSZ_types() -''' + return committee_cache[param_hash]''' def remove_for_phase1(functions: Dict[str, str]): @@ -128,23 +117,10 @@ def remove_for_phase1(functions: Dict[str, str]): functions[key] = "\n".join(lines) -def strip_comments(raw: str) -> str: - comment_line_regex = re.compile(r'^\s+# ') - lines = raw.split('\n') - out = [] - for line in lines: - if not comment_line_regex.match(line): - if ' #' in line: - line = line[:line.index(' #')] - out.append(line) - return '\n'.join(out) - - def objects_to_spec(functions: Dict[str, str], custom_types: Dict[str, str], constants: Dict[str, str], ssz_objects: Dict[str, str], - inserts: Dict[str, str], imports: Dict[str, str], ) -> str: """ @@ -169,27 +145,17 @@ def objects_to_spec(functions: Dict[str, str], constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) - ssz_objects_reinitialization_spec = ( - 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' - + '\n\n '.join([strip_comments(re.sub(r'\n\n', r'\n', re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]))) - for value in ssz_objects.values()]) - + '\n\n' - + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) - ) spec = ( imports + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS + '\n\n' + constants_spec - + '\n\n\n' + ssz_objects_instantiation_spec + + '\n\n' + CONFIG_LOADER + + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec + '\n' + SUNDRY_FUNCTIONS - + '\n\n' + ssz_objects_reinitialization_spec + '\n' ) - # Handle @inserts - for key, value in inserts.items(): - spec = re.sub('[ ]*# %s\\n' % key, value, spec) return spec @@ -242,32 +208,22 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str] and returns the newer versions of the objects in dependency order. """ for key, value in new_objects.items(): - if key in old_objects: - # add proper spacing - old_objects[key] = old_objects[key] + "\n\n" - lines = value.split("\n") - value = "\n".join([lines[0] + " # noqa: F811"] + lines[1:]) - old_objects[key] = old_objects.get(key, '') + value + old_objects[key] = value dependency_order_ssz_objects(old_objects, custom_types) return old_objects -# inserts are handeled the same way as functions -combine_inserts = combine_functions - - def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: """ Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. """ - functions0, custom_types0, constants0, ssz_objects0, inserts0 = spec0 - functions1, custom_types1, constants1, ssz_objects1, inserts1 = spec1 + functions0, custom_types0, constants0, ssz_objects0 = spec0 + functions1, custom_types1, constants1, ssz_objects1 = spec1 functions = combine_functions(functions0, functions1) custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) - inserts = combine_inserts(inserts0, inserts1) - return functions, custom_types, constants, ssz_objects, inserts + return functions, custom_types, constants, ssz_objects def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, diff --git a/scripts/function_puller.py b/scripts/function_puller.py index b30e5b75cd..1a134007e0 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -3,8 +3,6 @@ FUNCTION_REGEX = r'^def [\w_]*' -BEGIN_INSERT_REGEX = r'# begin insert ' -END_INSERT_REGEX = r'# end insert' SpecObject = NewType('SpecObjects', Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]) @@ -15,22 +13,18 @@ def get_spec(file_name: str) -> SpecObject: functions = {function_name: function_code} constants= {constant_name: constant_code} ssz_objects= {object_name: object} - inserts= {insert_tag: code to be inserted} Note: This function makes heavy use of the inherent ordering of dicts, if this is not supported by your python version, it will not work. """ pulling_from = None # line number of start of latest object current_name = None # most recent section title - insert_name = None # stores the label of the current insert object - functions = {} - constants = {} - ssz_objects = {} - inserts = {} + functions: Dict[str, str] = {} + constants: Dict[str, str] = {} + ssz_objects: Dict[str, str] = {} function_matcher = re.compile(FUNCTION_REGEX) - inserts_matcher = re.compile(BEGIN_INSERT_REGEX) is_ssz = False - custom_types = {} + custom_types: Dict[str, str] = {} for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -40,15 +34,6 @@ def get_spec(file_name: str) -> SpecObject: pulling_from = linenum + 1 elif line[:3] == '```': pulling_from = None - elif inserts_matcher.match(line) is not None: - # Find @insert names - insert_name = re.search(r'@[\w]*', line).group(0) - elif insert_name is not None: - # In insert mode, either the next line is more code, or the end of the insert - if re.match(END_INSERT_REGEX, line) is not None: - insert_name = None - else: - inserts[insert_name] = inserts.get(insert_name, '') + line + '\n' else: # Handle function definitions & ssz_objects if pulling_from is not None: @@ -84,4 +69,4 @@ def get_spec(file_name: str) -> SpecObject: constants[row[0]] = row[1].replace('**TBD**', '2**32') elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] - return functions, custom_types, constants, ssz_objects, inserts + return SpecObject((functions, custom_types, constants, ssz_objects)) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 63c2071a79..919c2a36a5 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -157,7 +157,7 @@ class AttestationCustodyBitWrapper(Container): ### New extended `PendingAttestation` ```python -class PendingAttestation(phase0.PendingAttestation): +class PendingAttestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData inclusion_delay: Slot @@ -168,7 +168,7 @@ class PendingAttestation(phase0.PendingAttestation): ### New extended `Validator` ```python -class Validator(phase0.Validator): +class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals effective_balance: Gwei # Balance at stake @@ -189,7 +189,7 @@ class Validator(phase0.Validator): ### New extended `BeaconBlockBody` ```python -class BeaconBlockBody(phase0.BeaconBlockBody): +class BeaconBlockBody(Container): randao_reveal: BLSSignature eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data @@ -218,7 +218,7 @@ class BeaconBlockBody(phase0.BeaconBlockBody): Note that the `body` has a new `BeaconBlockBody` definition. ```python -class BeaconBlock(phase0.BeaconBlock): +class BeaconBlock(Container): slot: Slot parent_root: Hash state_root: Hash @@ -231,7 +231,7 @@ class BeaconBlock(phase0.BeaconBlock): Note that aside from the new additions, `Validator` and `PendingAttestation` have new definitions. ```python -class BeaconState(phase0.BeaconState): +class BeaconState(Container): # Versioning genesis_time: uint64 slot: Slot @@ -270,7 +270,7 @@ class BeaconState(phase0.BeaconState): custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], + exposed_derived_secrets: Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` @@ -610,7 +610,7 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest # Apply proposer reward and cost beacon_proposer_index = get_beacon_proposer_index(state) estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) - proposer_reward = estimated_attester_reward // PROPOSER_REWARD_COEFFICIENT + proposer_reward = estimated_attester_reward // PROPOSER_REWARD_QUOTIENT increase_balance(state, beacon_proposer_index, proposer_reward) states_slots_lengths = zip( block_body.shard_transition.shard_states, @@ -659,19 +659,19 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB assert len(block_body.light_client_signature_bitfield) == len(committee) total_reward = Gwei(0) signer_keys = [] - for i, bit in enumerate(block_body.light_client_signature_bitfield): - if bit: + for i, participant_bit in enumerate(block_body.light_client_signature_bitfield): + if participant_bit: signer_keys.append(state.validators[committee[i]].pubkey) increase_balance(state, committee[i], get_base_reward(state, committee[i])) total_reward += get_base_reward(state, committee[i]) - increase_balance(state, get_beacon_proposer_index(state), total_reward // PROPOSER_REWARD_COEFFICIENT) + increase_balance(state, get_beacon_proposer_index(state), total_reward // PROPOSER_REWARD_QUOTIENT) assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), message_hash=get_block_root_at_slot(state, state.slot - 1), signature=block_body.light_client_signature, - domain=DOMAIN_LIGHT_CLIENT, + domain=DOMAIN_LIGHT_CLIENT ) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 6c49ee4701..7b554720dc 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -34,14 +34,12 @@ - [Helpers](#helpers) - [`ceillog2`](#ceillog2) - [`is_valid_merkle_branch_with_mixin`](#is_valid_merkle_branch_with_mixin) - - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`legendre_bit`](#legendre_bit) - [`custody_subchunkify`](#custody_subchunkify) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) - - [`replace_empty_or_append`](#replace_empty_or_append) - [Per-block processing](#per-block-processing) - [Operations](#operations) - [Custody key reveals](#custody-key-reveals) @@ -168,45 +166,6 @@ class CustodyBitChallenge(Container): signature: BLSSignature ``` -#### `CustodyChunkChallengeRecord` - -```python -class CustodyChunkChallengeRecord(Container): - challenge_index: uint64 - challenger_index: ValidatorIndex - responder_index: ValidatorIndex - inclusion_epoch: Epoch - data_root: Root - depth: uint64 - chunk_index: uint64 -``` - -#### `CustodyBitChallengeRecord` - -```python -class CustodyBitChallengeRecord(Container): - challenge_index: uint64 - challenger_index: ValidatorIndex - responder_index: ValidatorIndex - inclusion_epoch: Epoch - data_root: Root - chunk_count: uint64 - chunk_bits_merkle_root: Root - responder_key: BLSSignature -``` - -#### `CustodyResponse` - -```python -class CustodyResponse(Container): - challenge_index: uint64 - chunk_index: uint64 - chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32, PLACEHOLDER] - chunk_bits_branch: List[Bytes32, PLACEHOLDER] - chunk_bits_leaf: Bytes32 -``` - ### New Beacon Chain operations #### `CustodyKeyReveal` @@ -266,13 +225,6 @@ def is_valid_merkle_branch_with_mixin(leaf: Bytes32, return value == root ``` -### `get_crosslink_chunk_count` - -```python -def get_custody_chunk_count(crosslink: Crosslink) -> int: - crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, crosslink.end_epoch - crosslink.start_epoch) - return crosslink_length * CHUNKS_PER_EPOCH -``` ### `legendre_bit` @@ -354,17 +306,6 @@ def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epo return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD ``` -### `replace_empty_or_append` - -```python -def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int: - for i in range(len(list)): - if is_zero(list[i]): - list[i] = new_element - return i - list.append(new_element) - return len(list) - 1 -``` ## Per-block processing @@ -526,7 +467,7 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge record.chunk_index != challenge.chunk_index ) # Verify depth - depth = ceillog2(get_custody_chunk_count(challenge.attestation.data.crosslink)) + depth = 123 # TODO assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -611,24 +552,13 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) # Verify the chunk count - chunk_count = get_custody_chunk_count(attestation.data.crosslink) + chunk_count = 123 # TODO assert chunk_count == len(challenge.chunk_bits) # Verify custody bit is incorrect committee = get_beacon_committee(state, epoch, shard) custody_bit = attestation.custody_bits[committee.index(challenge.responder_index)] assert custody_bit != get_chunk_bits_root(challenge.chunk_bits) - # Add new bit challenge record - new_record = CustodyBitChallengeRecord( - challenge_index=state.custody_challenge_index, - challenger_index=challenge.challenger_index, - responder_index=challenge.responder_index, - inclusion_epoch=get_current_epoch(state), - data_root=attestation.data.crosslink.data_root, - chunk_count=chunk_count, - chunk_bits_merkle_root=hash_tree_root(challenge.chunk_bits), - responder_key=challenge.responder_key, - ) - replace_empty_or_append(state.custody_bit_challenge_records, new_record) + # TODO: immediate processing of challenge? state.custody_challenge_index += 1 # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index 62c8966691..ff258366ae 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -99,7 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: next_light_committee=CompactCommittee(), # Custody game custody_challenge_index=0, - exposed_derived_secrets=Vector[List[ValidatorIndex, PLACEHOLDER], + exposed_derived_secrets=Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() ) post.current_light_committee = get_light_client_committee(post, post.epoch) diff --git a/test_libs/pyspec/eth2spec/config/__init__.py b/test_libs/pyspec/eth2spec/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/config/apply_config.py b/test_libs/pyspec/eth2spec/config/apply_config.py new file mode 100644 index 0000000000..768abba648 --- /dev/null +++ b/test_libs/pyspec/eth2spec/config/apply_config.py @@ -0,0 +1,19 @@ +from preset_loader import loader +from typing import Dict, Any + +presets: Dict[str, Any] = {} + + +# Access to overwrite spec constants based on configuration +def apply_constants_preset(spec_globals: Dict[str, Any]) -> None: + global presets + for k, v in presets.items(): + if k.startswith('DOMAIN_'): + spec_globals[k] = spec_globals['DomainType'](v) # domain types are defined as bytes in the configs + else: + spec_globals[k] = v + + +def load_presets(configs_path, config_name): + global presets + presets = loader.load_presets(configs_path, config_name) diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py index 5713c34706..a0a200f0bb 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -1,5 +1,4 @@ -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.config import apply_config # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. @@ -33,7 +32,4 @@ def pytest_addoption(parser): @fixture(autouse=True) def config(request): config_name = request.config.getoption("--config") - from preset_loader import loader - presets = loader.load_presets('../../configs/', config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + apply_config.load_presets('../../configs/', config_name) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 3177cd0b84..0fb46aa50b 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -1,5 +1,5 @@ from eth2spec.phase0 import spec as spec_phase0 -# from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls from .helpers.genesis import create_genesis_state @@ -8,6 +8,11 @@ from typing import Any, Callable, Sequence +from importlib import reload + +reload(spec_phase0) +reload(spec_phase1) + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): @@ -191,9 +196,7 @@ def wrapper(*args, **kw): if 'phase0' in run_phases: ret = run_with_spec_version(spec_phase0, *args, **kw) if 'phase1' in run_phases: - # temporarily disable phase 1 tests - return - # ret = run_with_spec_version(spec_phase1, *args, **kw) + ret = run_with_spec_version(spec_phase1, *args, **kw) return ret return wrapper return decorator From 1a1c3773f935d0c8634d49dd55fc7c74edcd16cf Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 Nov 2019 04:15:15 +0100 Subject: [PATCH 062/111] implement custody game revamp for new shard proposal: simplifications and immediate processing, since custody data is bounded --- specs/core/1_beacon-chain.md | 25 +-- specs/core/1_custody-game.md | 383 ++++++++--------------------------- 2 files changed, 99 insertions(+), 309 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 919c2a36a5..0216369672 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -39,11 +39,11 @@ Configuration is not namespaced. Instead it is strictly an extension; | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | -| `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | -| `BLOCK_SIZE_TARGET` | `3 * 2**16` (= 196,608) | | +| `SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | +| `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | +| `EMPTY_CHUNK_ROOT` | `hash_tree_root(ByteVector[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | @@ -62,7 +62,7 @@ class ShardBlockWrapper(Container): shard_parent_root: Hash beacon_parent_root: Hash slot: Slot - body: BytesN[SHARD_BLOCK_CHUNK_SIZE] + body: ByteVector[MAX_SHARD_BLOCK_SIZE] signature: BLSSignature ``` @@ -202,8 +202,7 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game - custody_chunk_challenges: List[CustodyChunkChallenge, MAX_CUSTODY_CHUNK_CHALLENGES] - custody_bit_challenges: List[CustodyBitChallenge, MAX_CUSTODY_BIT_CHALLENGES] + custody_slashings: List[CustodySlashing, MAX_CUSTODY_SLASHINGS] custody_key_reveals: List[CustodyKeyReveal, MAX_CUSTODY_KEY_REVEALS] early_derived_secret_reveals: List[EarlyDerivedSecretReveal, MAX_EARLY_DERIVED_SECRET_REVEALS] # Shards @@ -266,8 +265,6 @@ class BeaconState(Container): current_light_committee: CompactCommittee next_light_committee: CompactCommittee # Custody game - # TODO: custody game refactor, no challenge-records, immediate processing. - custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS exposed_derived_secrets: Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], @@ -317,7 +314,9 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ```python def chunks_to_body_root(chunks): - return hash_tree_root(chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))) + return hash_tree_root(Vector[Hash, MAX_SHARD_BLOCK_CHUNKS]( + chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) + )) ``` ### Beacon state accessors @@ -375,11 +374,13 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ```python def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: - if length > BLOCK_SIZE_TARGET: - delta = prev_gasprice * (length - BLOCK_SIZE_TARGET) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT + if length > TARGET_SHARD_BLOCK_SIZE: + delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return min(prev_gasprice + delta, MAX_GASPRICE) else: - delta = prev_gasprice * (BLOCK_SIZE_TARGET - length) // BLOCK_SIZE_TARGET // GASPRICE_ADJUSTMENT_COEFFICIENT + delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 7b554720dc..7ae1c51702 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -12,43 +12,29 @@ - [Terminology](#terminology) - [Constants](#constants) - [Misc](#misc) - - [Custody game parameters](#custody-game-parameters) - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) - - [Custody objects](#custody-objects) - - [`CustodyChunkChallenge`](#custodychunkchallenge) - - [`CustodyBitChallenge`](#custodybitchallenge) - - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) - - [`CustodyBitChallengeRecord`](#custodybitchallengerecord) - - [`CustodyResponse`](#custodyresponse) - [New beacon operations](#new-beacon-operations) + - [`CustodySlashing`](#custody-slashing) - [`CustodyKeyReveal`](#custodykeyreveal) - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - - [Phase 0 container updates](#phase-0-container-updates) - - [`Validator`](#validator) - - [`BeaconState`](#beaconstate) - - [`BeaconBlockBody`](#beaconblockbody) - [Helpers](#helpers) - - [`ceillog2`](#ceillog2) - - [`is_valid_merkle_branch_with_mixin`](#is_valid_merkle_branch_with_mixin) - [`legendre_bit`](#legendre_bit) - - [`custody_subchunkify`](#custody_subchunkify) - - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - - [`get_chunk_bits_root`](#get_chunk_bits_root) + - [`get_custody_atoms`](#get_custody_atoms) + - [`compute_custody_bit`](#compute_custody_bit) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) - [Per-block processing](#per-block-processing) - - [Operations](#operations) + - [Custody Game Operations](#custody-game-operations) - [Custody key reveals](#custody-key-reveals) - [Early derived secret reveals](#early-derived-secret-reveals) - - [Chunk challenges](#chunk-challenges) - - [Bit challenges](#bit-challenges) - - [Custody responses](#custody-responses) + - [Custody Slashings](#custody-slashings) - [Per-epoch processing](#per-epoch-processing) - - [Handling of custody-related deadlines](#handling-of-custody-related-deadlines) + - [Handling of reveal deadlines](#handling-of-reveal-deadlines) + - [Final updates](#final-updates) @@ -56,46 +42,19 @@ This document details the beacon chain additions and changes in Phase 1 of Ethereum 2.0 to support the shard data custody game, building upon the [Phase 0](0_beacon-chain.md) specification. -## Terminology - -- **Custody game**— -- **Custody period**— -- **Custody chunk**— -- **Custody chunk bit**— -- **Custody chunk challenge**— -- **Custody bit**— -- **Custody bit challenge**— -- **Custody key**— -- **Custody key reveal**— -- **Custody key mask**— - ## Constants ### Misc -| Name | Value | +| Name | Value | Unit | | - | - | | `BLS12_381_Q` | `4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787` | | `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | -| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | - -### Custody game parameters - -| Name | Value | -| - | - | -| `BYTES_PER_SHARD_BLOCK` | `2**14` (= 16,384) | -| `BYTES_PER_CUSTODY_CHUNK` | `2**9` (= 512) | -| `BYTES_PER_CUSTODY_SUBCHUNK` | `48` | -| `CHUNKS_PER_EPOCH` | `2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK` | -| `MAX_CUSTODY_CHUNKS` | `MAX_EPOCHS_PER_CROSSLINK * CHUNKS_PER_EPOCH` | -| `CUSTODY_DATA_DEPTH` | `ceillog2(MAX_CUSTODY_CHUNKS) + 1` | -| `CUSTODY_CHUNK_BIT_DEPTH` | `ceillog2(MAX_EPOCHS_PER_CROSSLINK * CHUNKS_PER_EPOCH // 256) + 2` | +| `BYTES_PER_CUSTODY_ATOM` | `48` | bytes | ### Time parameters | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 2,048) | epochs | ~9 days | -| `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | | `RANDAO_PENALTY_EPOCHS` | `2**1` (= 2) | epochs | 12.8 minutes | | `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` | epochs | ~73 days | | `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | @@ -108,8 +67,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `MAX_CUSTODY_KEY_REVEALS` | `2**4` (= 16) | | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | -| `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | -| `MAX_CUSTODY_BIT_CHALLENGES` | `2**2` (= 4) | +| `MAX_CUSTODY_SLASHINGS` | `1` | ### Reward and penalty quotients @@ -123,51 +81,29 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | +| `DOMAIN_CUSTODY_BIT_SLASHING` | `6` | ## Data structures -### Custody objects - -#### `Crosslink` - -*Note*: Crosslinks have been removed in the phase 1 redesign. This is a placeholder until the custody game is revamped. - -```python -class Crosslink(Container): - shard: uint64 - parent_root: Hash - # Crosslinking data - start_epoch: Epoch - end_epoch: Epoch - data_root: Hash -``` - - -#### `CustodyChunkChallenge` - -```python -class CustodyChunkChallenge(Container): - responder_index: ValidatorIndex - attestation: Attestation - chunk_index: uint64 -``` +### New Beacon Chain operations -#### `CustodyBitChallenge` +#### `CustodySlashing` ```python -class CustodyBitChallenge(Container): - responder_index: ValidatorIndex +class CustodySlashing(Container): + # Attestation.custody_bits[data_index][committee.index(malefactor_index)] is the target custody bit to check. + # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. + data_index: uint64 + malefactor_index: ValidatorIndex + malefactor_key: BLSSignature + whistleblower_index: ValidatorIndex + shard_transition: ShardTransition attestation: Attestation - challenger_index: ValidatorIndex - responder_key: BLSSignature - chunk_bits: Bitlist[MAX_CUSTODY_CHUNKS] + data: ByteList[MAX_SHARD_BLOCK_SIZE] signature: BLSSignature ``` -### New Beacon Chain operations - #### `CustodyKeyReveal` ```python @@ -199,33 +135,6 @@ class EarlyDerivedSecretReveal(Container): ## Helpers -### `ceillog2` - -```python -def ceillog2(x: uint64) -> int: - return (x - 1).bit_length() -``` - -### `is_valid_merkle_branch_with_mixin` - -```python -def is_valid_merkle_branch_with_mixin(leaf: Bytes32, - branch: Sequence[Bytes32], - depth: uint64, - index: uint64, - root: Root, - mixin: uint64) -> bool: - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(branch[i] + value) - else: - value = hash(value + branch[i]) - value = hash(value + mixin.to_bytes(32, "little")) - return value == root -``` - - ### `legendre_bit` Returns the Legendre symbol `(a/q)` normalizes as a bit (i.e. `((a/q) + 1) // 2`). In a production implementation, a well-optimized library (e.g. GMP) should be used for this. @@ -255,39 +164,29 @@ def legendre_bit(a: int, q: int) -> int: return 0 ``` -### `custody_subchunkify` +### `custody_atoms` -Given one proof of custody chunk, returns the proof of custody subchunks of the correct sizes. +Given one set of data, return the custody atoms: each atom will be combined with one legendre bit. ```python -def custody_subchunkify(bytez: bytes) -> Sequence[bytes]: - bytez += b'\x00' * (-len(bytez) % BYTES_PER_CUSTODY_SUBCHUNK) - return [bytez[i:i + BYTES_PER_CUSTODY_SUBCHUNK] - for i in range(0, len(bytez), BYTES_PER_CUSTODY_SUBCHUNK)] +def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: + bytez += b'\x00' * (-len(bytez) % BYTES_PER_CUSTODY_ATOM) # right-padding + return [bytez[i:i + BYTES_PER_CUSTODY_ATOM] + for i in range(0, len(bytez), BYTES_PER_CUSTODY_ATOM)] ``` -### `get_custody_chunk_bit` +### `compute_custody_bit` ```python -def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: +def compute_custody_bit(key: BLSSignature, data: bytes) -> bool: full_G2_element = bls_signature_to_G2(key) s = full_G2_element[0].coeffs - bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(subchunk, "little"), BLS12_381_Q) - for i, subchunk in enumerate(custody_subchunkify(chunk))] - + bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(atom, "little"), BLS12_381_Q) + for i, atom in enumerate(get_custody_atoms(data))] + # XOR all atom bits return bool(sum(bits) % 2) ``` -### `get_chunk_bits_root` - -```python -def get_chunk_bits_root(chunk_bits: Bitlist[MAX_CUSTODY_CHUNKS]) -> bit: - aggregated_bits = 0 - for i, b in enumerate(chunk_bits): - aggregated_bits += 2**i * b - return legendre_bit(aggregated_bits, BLS12_381_Q) -``` - ### `get_randao_epoch_for_custody_period` ```python @@ -319,8 +218,7 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - for_ops(body.custody_key_reveals, process_custody_key_reveal) for_ops(body.early_derived_secret_reveals, process_early_derived_secret_reveal) - for_ops(body.custody_chunk_challenges, process_chunk_challenge) - for_ops(body.custody_bit_challenges, process_bit_challenge) + for_ops(body.custody_slashings, process_custody_slashing) ``` #### Custody key reveals @@ -367,7 +265,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> # Process reveal revealer.next_custody_secret_to_reveal += 1 - # Reward Block Preposer + # Reward Block Proposer proposer_index = get_beacon_proposer_index(state) increase_balance( state, @@ -446,190 +344,81 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived state.exposed_derived_secrets[derived_secret_location].append(reveal.revealed_index) ``` -#### Chunk challenges +#### Custody Slashings ```python -def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: +def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashing) -> None: + attestation = custody_slashing.attestation + + # Any signed custody-slashing should result in at least one slashing. + # If the custody bits are valid, then the claim itself is slashed. + malefactor = state.validators[custody_slashing.malefactor_index] + whistleblower = state.validators[custody_slashing.whistleblower_index] + domain = get_domain(state, DOMAIN_CUSTODY_BIT_SLASHING, get_current_epoch(state)) + assert bls_verify(whistleblower.pubkey, signing_root(custody_slashing), custody_slashing.signature, domain) + # Verify that the whistleblower is slashable + assert is_slashable_validator(whistleblower, get_current_epoch(state)) + # Verify that the claimed malefactor is slashable + assert is_slashable_validator(malefactor, get_current_epoch(state)) + # Verify the attestation - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) - # Verify it is not too late to challenge - assert (compute_epoch_at_slot(challenge.attestation.data.slot) - >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY) - responder = state.validators[challenge.responder_index] - assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY - # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) - assert challenge.responder_index in attesters - # Verify the challenge is not a duplicate - for record in state.custody_chunk_challenge_records: - assert ( - record.data_root != challenge.attestation.data.crosslink.data_root or - record.chunk_index != challenge.chunk_index - ) - # Verify depth - depth = 123 # TODO - assert challenge.chunk_index < 2**depth - # Add new chunk challenge record - new_record = CustodyChunkChallengeRecord( - challenge_index=state.custody_challenge_index, - challenger_index=get_beacon_proposer_index(state), - responder_index=challenge.responder_index, - inclusion_epoch=get_current_epoch(state), - data_root=challenge.attestation.data.crosslink.data_root, - depth=depth, - chunk_index=challenge.chunk_index, - ) - replace_empty_or_append(state.custody_chunk_challenge_records, new_record) + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - state.custody_challenge_index += 1 - # Postpone responder withdrawability - responder.withdrawable_epoch = FAR_FUTURE_EPOCH -``` + # TODO: custody_slashing.data is not chunked like shard blocks yet, result is lots of padding. -TODO: immediate challenge processing, no state records. + # TODO: can do a single combined merkle proof of data being attested. + # Verify the shard transition is indeed attested by the attestation + shard_transition = custody_slashing.shard_transition + assert hash_tree_root(shard_transition) == attestation.shard_transition_root + # Verify that the provided data matches the shard-transition + shard_chunk_roots = shard_transition.shard_data_roots[custody_slashing.data_index] + assert hash_tree_root(custody_slashing.data) == chunks_to_body_root(shard_chunk_roots) -```python -def process_chunk_challenge_response(state: BeaconState, - response: CustodyResponse, - challenge: CustodyChunkChallengeRecord) -> None: - # Verify chunk index - assert response.chunk_index == challenge.chunk_index - # Verify bit challenge data is null - assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == Hash() - # Verify minimum delay - assert get_current_epoch(state) >= challenge.inclusion_epoch + MAX_SEED_LOOKAHEAD - # Verify the chunk matches the crosslink data root - assert is_valid_merkle_branch( - leaf=hash_tree_root(response.chunk), - branch=response.data_branch, - depth=challenge.depth, - index=response.chunk_index, - root=challenge.data_root, - ) - # Clear the challenge - records = state.custody_chunk_challenge_records - records[records.index(challenge)] = CustodyChunkChallengeRecord() - # Reward the proposer - proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) -``` - -#### Bit challenges - -```python -def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: - attestation = challenge.attestation - epoch = attestation.data.target.epoch - shard = attestation.data.crosslink.shard - - # Verify challenge signature - challenger = state.validators[challenge.challenger_index] - domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) - # TODO incorrect hash-tree-root, but this changes with phase 1 PR #1483 - assert bls_verify(challenger.pubkey, hash_tree_root(challenge), challenge.signature, domain) - # Verify challenger is slashable - assert is_slashable_validator(challenger, get_current_epoch(state)) - # Verify attestation - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # Verify attestation is eligible for challenging - responder = state.validators[challenge.responder_index] - assert get_current_epoch(state) <= get_randao_epoch_for_custody_period( - get_custody_period_for_validator(challenge.responder_index, epoch), - challenge.responder_index - ) + 2 * EPOCHS_PER_CUSTODY_PERIOD + responder.max_reveal_lateness - - # Verify the responder participated in the attestation + # Verify existence of claimed malefactor attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - assert challenge.responder_index in attesters - # Verifier challenger is not already challenging - for record in state.custody_bit_challenge_records: - assert record.challenger_index != challenge.challenger_index - # Verify the responder custody key - epoch_to_sign = get_randao_epoch_for_custody_period( - get_custody_period_for_validator(challenge.responder_index, epoch), - challenge.responder_index, - ) - domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) - # Verify the chunk count - chunk_count = 123 # TODO - assert chunk_count == len(challenge.chunk_bits) - # Verify custody bit is incorrect - committee = get_beacon_committee(state, epoch, shard) - custody_bit = attestation.custody_bits[committee.index(challenge.responder_index)] - assert custody_bit != get_chunk_bits_root(challenge.chunk_bits) - # TODO: immediate processing of challenge? - state.custody_challenge_index += 1 - # Postpone responder withdrawability - responder.withdrawable_epoch = FAR_FUTURE_EPOCH + assert custody_slashing.malefactor_index in attesters + + # Get the custody bit + custody_bits = attestation.custody_bits[custody_slashing.data_index] + claimed_custody_bit = custody_bits[attesters.index(custody_slashing.malefactor_index)] + + # Compute the custody bit + computed_custody_bit = compute_custody_bit(custody_slashing.data) + + # Verify the claim + if claimed_custody_bit != computed_custody_bit: + # Slash the malefactor, reward the other committee members + slash_validator(state, custody_slashing.malefactor_index) + whistleblower_reward = Gwei(malefactor.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) // len(attesters - 1) + for attester_index in attesters: + if attester_index != custody_slashing.malefactor_index: + increase_balance(state, attester_index, whistleblower_reward) + # No special whisteblower reward: it is expected to be an attester. Others are free to slash too however. + else: + # The claim was false, the custody bit was correct. Slash the whistleblower that induced this work. + slash_validator(state, custody_slashing.whistleblower_index) ``` -TODO: immediate challenge processing, no state records. - -```python -def process_bit_challenge_response(state: BeaconState, - response: CustodyResponse, - challenge: CustodyBitChallengeRecord) -> None: - # Verify chunk index - assert response.chunk_index < challenge.chunk_count - # Verify responder has not been slashed - responder = state.validators[challenge.responder_index] - assert not responder.slashed - # Verify the chunk matches the crosslink data root - assert is_valid_merkle_branch( - leaf=hash_tree_root(response.chunk), - branch=response.data_branch, - depth=ceillog2(challenge.chunk_count), - index=response.chunk_index, - root=challenge.data_root, - ) - # Verify the chunk bit leaf matches the challenge data - assert is_valid_merkle_branch_with_mixin( - leaf=hash_tree_root(response.chunk_bits_leaf), - branch=response.chunk_bits_branch, - depth=ceillog2(MAX_CUSTODY_CHUNKS // 256), - index=response.chunk_index // 256, - root=challenge.chunk_bits_merkle_root, - mixin=challenge.chunk_count, - ) - # Verify the chunk bit does not match the challenge chunk bit - assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != response.chunk_bits_leaf[response.chunk_index % 256]) - # Clear the challenge - records = state.custody_bit_challenge_records - records[records.index(challenge)] = CustodyBitChallengeRecord() - # Slash challenger - slash_validator(state, challenge.challenger_index, challenge.responder_index) -``` ## Per-epoch processing -### Handling of custody-related deadlines +### Handling of reveal deadlines Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: ```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validators): - deadline = validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - if get_custody_period_for_validator(state, ValidatorIndex(index)) > deadline: + if get_custody_period_for_validator(state, ValidatorIndex(index)) > validator.next_custody_secret_to_reveal: slash_validator(state, ValidatorIndex(index)) ``` +### Final updates + After `process_final_updates(state)`, additional updates are made for the custody game: ```python def process_custody_final_updates(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) # Clean up exposed RANDAO key reveals - state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] - # Reset withdrawable epochs if challenge records are empty - records = state.custody_chunk_challenge_records + state.custody_bit_challenge_records - validator_indices_in_records = set( - [record.challenger_index for record in records] + [record.responder_index for record in records] - ) - for index, validator in enumerate(state.validators): - if index not in validator_indices_in_records: - if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: - validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) + state.exposed_derived_secrets[get_current_epoch(state) % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] ``` From 41be2ed3ce84fff54a9a0a05b8736b8583687810 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 Nov 2019 04:43:32 +0100 Subject: [PATCH 063/111] bugfix custody bit index lookup + lint fixes --- scripts/build_spec.py | 25 +++++++++++-------------- specs/core/0_beacon-chain.md | 2 +- specs/core/1_beacon-chain.md | 14 +++++++------- specs/core/1_custody-game.md | 13 +++++++------ 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3352864378..6dfc4e6b28 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -15,7 +15,7 @@ PHASE0_IMPORTS = '''from eth2spec.config.apply_config import apply_constants_preset from typing import ( - Dict, Set, Sequence, Tuple, Optional + Any, Callable, Dict, Set, Sequence, Tuple, Optional ) from dataclasses import ( @@ -40,7 +40,7 @@ PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.apply_config import apply_constants_preset from typing import ( - Dict, Set, Sequence, NewType, Tuple, Union, + Any, Callable, Dict, Set, Sequence, NewType, Tuple, Union, ) from math import ( log2, @@ -110,13 +110,6 @@ def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore return committee_cache[param_hash]''' -def remove_for_phase1(functions: Dict[str, str]): - for key, value in functions.items(): - lines = value.split("\n") - lines = filter(lambda s: "[to be removed in phase 1]" not in s, lines) - functions[key] = "\n".join(lines) - - def objects_to_spec(functions: Dict[str, str], custom_types: Dict[str, str], constants: Dict[str, str], @@ -172,10 +165,10 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'ByteList', 'ByteVector' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', - 'bytes', 'byte', 'ByteVector' # to be removed after updating spec doc + 'bytes', 'byte', 'Bytes', 'BytesN' # to be removed after updating spec doc ] @@ -209,7 +202,6 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str] """ for key, value in new_objects.items(): old_objects[key] = value - dependency_order_ssz_objects(old_objects, custom_types) return old_objects @@ -226,6 +218,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: return functions, custom_types, constants, ssz_objects +def dependency_order_spec(objs: SpecObject): + functions, custom_types, constants, ssz_objects = objs + dependency_order_ssz_objects(ssz_objects, custom_types) + + def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, v_guide_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) @@ -234,6 +231,7 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, spec_objects = phase0_spec for value in [fork_choice_spec, v_guide]: spec_objects = combine_spec_objects(spec_objects, value) + dependency_order_spec(spec_objects) spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: @@ -259,11 +257,10 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, phase1_fork_sourcefile, ) all_spescs = [get_spec(spec) for spec in all_sourcefiles] - for spec in all_spescs: - remove_for_phase1(spec[0]) spec_objects = all_spescs[0] for value in all_spescs[1:]: spec_objects = combine_spec_objects(spec_objects, value) + dependency_order_spec(spec_objects) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 32c10c4ecd..7b182e0879 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1419,7 +1419,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - def for_ops(operations, fn): + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: for operation in operations: fn(state, operation) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 0216369672..3f964641c9 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -39,11 +39,11 @@ Configuration is not namespaced. Instead it is strictly an extension; | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | +| `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | | `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `EMPTY_CHUNK_ROOT` | `hash_tree_root(ByteVector[SHARD_BLOCK_CHUNK_SIZE]())` | | +| `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | @@ -62,7 +62,7 @@ class ShardBlockWrapper(Container): shard_parent_root: Hash beacon_parent_root: Hash slot: Slot - body: ByteVector[MAX_SHARD_BLOCK_SIZE] + body: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] signature: BLSSignature ``` @@ -315,7 +315,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ```python def chunks_to_body_root(chunks): return hash_tree_root(Vector[Hash, MAX_SHARD_BLOCK_CHUNKS]( - chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) + chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) )) ``` @@ -376,11 +376,11 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: if length > TARGET_SHARD_BLOCK_SIZE: delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) - // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return min(prev_gasprice + delta, MAX_GASPRICE) else: delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) - // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` @@ -465,7 +465,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - def for_ops(operations, fn): + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: for operation in operations: fn(state, operation) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 7ae1c51702..ebba75a7ff 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -100,7 +100,7 @@ class CustodySlashing(Container): whistleblower_index: ValidatorIndex shard_transition: ShardTransition attestation: Attestation - data: ByteList[MAX_SHARD_BLOCK_SIZE] + data: Bytes[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] signature: BLSSignature ``` @@ -178,13 +178,13 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: ### `compute_custody_bit` ```python -def compute_custody_bit(key: BLSSignature, data: bytes) -> bool: +def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: full_G2_element = bls_signature_to_G2(key) s = full_G2_element[0].coeffs bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(atom, "little"), BLS12_381_Q) for i, atom in enumerate(get_custody_atoms(data))] # XOR all atom bits - return bool(sum(bits) % 2) + return bit(sum(bits) % 2) ``` ### `get_randao_epoch_for_custody_period` @@ -212,7 +212,7 @@ def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epo ```python def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) -> None: - def for_ops(operations, fn): + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: for operation in operations: fn(state, operation) @@ -374,13 +374,14 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi shard_chunk_roots = shard_transition.shard_data_roots[custody_slashing.data_index] assert hash_tree_root(custody_slashing.data) == chunks_to_body_root(shard_chunk_roots) - # Verify existence of claimed malefactor + # Verify existence and participation of claimed malefactor attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert custody_slashing.malefactor_index in attesters # Get the custody bit custody_bits = attestation.custody_bits[custody_slashing.data_index] - claimed_custody_bit = custody_bits[attesters.index(custody_slashing.malefactor_index)] + committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) + claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] # Compute the custody bit computed_custody_bit = compute_custody_bit(custody_slashing.data) From 1623d40b6df8d9c03c691ad79413ec4eb8050084 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 Nov 2019 04:48:00 +0100 Subject: [PATCH 064/111] fix: check malefactor custody key --- specs/core/1_custody-game.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index ebba75a7ff..ed91bf8a08 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -378,13 +378,21 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert custody_slashing.malefactor_index in attesters + # Verify the malefactor custody key + epoch_to_sign = get_randao_epoch_for_custody_period( + get_custody_period_for_validator(state, custody_slashing.malefactor_index, attestation.data.target.epoch), + custody_slashing.malefactor_index, + ) + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + assert bls_verify(malefactor.pubkey, hash_tree_root(epoch_to_sign), custody_slashing.malefactor_key, domain) + # Get the custody bit custody_bits = attestation.custody_bits[custody_slashing.data_index] committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] # Compute the custody bit - computed_custody_bit = compute_custody_bit(custody_slashing.data) + computed_custody_bit = compute_custody_bit(custody_slashing.malefactor_key, custody_slashing.data) # Verify the claim if claimed_custody_bit != computed_custody_bit: From 72a9dfc43092967e47775cf62fc5e07947296a3f Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 Nov 2019 05:11:19 +0100 Subject: [PATCH 065/111] fix lint problems --- specs/core/1_beacon-chain.md | 40 ++++++++++++++++++++++-------------- specs/core/1_custody-game.md | 8 +++++--- specs/core/1_phase1-fork.md | 8 ++++---- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 3f964641c9..4fe49c92dc 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -34,18 +34,18 @@ Configuration is not namespaced. Instead it is strictly an extension; | - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | | `ACTIVE_SHARDS` | `2**6` (= 64) | -| `ONLINE_PERIOD` | `2**3` (= 8) | epochs | ~51 min | +| `ONLINE_PERIOD` | `Epoch(2**3)` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | -| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | -| `SHARD_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | +| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | +| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | | | `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | | | `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | -| `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `2**5` (= 32) | Gwei | | +| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | +| `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | `DOMAIN_LIGHT_CLIENT` | `192` | | | `DOMAIN_SHARD_COMMITTEE` | `192` | | @@ -313,7 +313,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid #### `chunks_to_body_root` ```python -def chunks_to_body_root(chunks): +def chunks_to_body_root(chunks: List[Hash, MAX_SHARD_BLOCK_CHUNKS]) -> Hash: return hash_tree_root(Vector[Hash, MAX_SHARD_BLOCK_CHUNKS]( chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) )) @@ -321,6 +321,16 @@ def chunks_to_body_root(chunks): ### Beacon state accessors +#### `get_previous_slot` + +```python +def get_previous_slot(state: BeaconState) -> Slot: + if state.slot > 0: + return Slot(state.slot - 1) + else: + return Slot(0) +``` + #### `get_online_validator_indices` ```python @@ -403,7 +413,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: - return [start_slot + x for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] + return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] ``` @@ -503,7 +513,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Correct data root count assert len(attestation.custody_bits) == len(get_offset_slots(state, state.shard_next_slots[shard])) # Correct parent block root - assert data.beacon_block_root == get_block_root_at_slot(state, state.slot - 1) + assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state)) # Type 2: delayed attestations else: assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH @@ -535,7 +545,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr if any(transition.shard_data_roots): headers.append(ShardSignableHeader( shard_parent_root=shard_parent_root, - parent_hash=get_block_root_at_slot(state, state.slot - 1), + parent_hash=get_block_root_at_slot(state, get_previous_slot(state)), slot=offset_slots[i], body_root=chunks_to_body_root(transition.shard_data_roots[i]) )) @@ -576,7 +586,7 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest # Process crosslinks online_indices = get_online_validator_indices(state) winners = set() - for shard in range(ACTIVE_SHARDS): + for shard in map(Shard, range(ACTIVE_SHARDS)): success = False # All attestations in the block for this shard this_shard_attestations = [ @@ -588,7 +598,7 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in this_shard_attestations]) for shard_transition_root in sorted(shard_transition_roots): - all_participants = set() + all_participants: Set[ValidatorIndex] = set() participating_attestations = [] for attestation in this_shard_attestations: participating_attestations.append(attestation) @@ -611,11 +621,11 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest # Apply proposer reward and cost beacon_proposer_index = get_beacon_proposer_index(state) estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) - proposer_reward = estimated_attester_reward // PROPOSER_REWARD_QUOTIENT + proposer_reward = Gwei(estimated_attester_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, beacon_proposer_index, proposer_reward) states_slots_lengths = zip( block_body.shard_transition.shard_states, - get_offset_slots(state, state.shard_next_slots[get_shard(attestation)]), + get_offset_slots(state, state.shard_next_slots[get_shard(state, attestation)]), block_body.shard_transition.shard_block_lengths ) for shard_state, slot, length in states_slots_lengths: @@ -666,11 +676,11 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB increase_balance(state, committee[i], get_base_reward(state, committee[i])) total_reward += get_base_reward(state, committee[i]) - increase_balance(state, get_beacon_proposer_index(state), total_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), - message_hash=get_block_root_at_slot(state, state.slot - 1), + message_hash=get_block_root_at_slot(state, get_previous_slot(state)), signature=block_body.light_client_signature, domain=DOMAIN_LIGHT_CLIENT ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index ed91bf8a08..61710cd5f8 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -380,7 +380,7 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi # Verify the malefactor custody key epoch_to_sign = get_randao_epoch_for_custody_period( - get_custody_period_for_validator(state, custody_slashing.malefactor_index, attestation.data.target.epoch), + get_custody_period_for_validator(custody_slashing.malefactor_index, attestation.data.target.epoch), custody_slashing.malefactor_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) @@ -398,7 +398,8 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi if claimed_custody_bit != computed_custody_bit: # Slash the malefactor, reward the other committee members slash_validator(state, custody_slashing.malefactor_index) - whistleblower_reward = Gwei(malefactor.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) // len(attesters - 1) + others_count = len(committee) - 1 + whistleblower_reward = Gwei(malefactor.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT // others_count) for attester_index in attesters: if attester_index != custody_slashing.malefactor_index: increase_balance(state, attester_index, whistleblower_reward) @@ -417,8 +418,9 @@ Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: ```python def process_reveal_deadlines(state: BeaconState) -> None: + epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - if get_custody_period_for_validator(state, ValidatorIndex(index)) > validator.next_custody_secret_to_reveal: + if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal: slash_validator(state, ValidatorIndex(index)) ``` diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index ff258366ae..a1e14ec354 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -64,9 +64,9 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: activation_epoch=phase0_validator.activation_eligibility_epoch, exit_epoch=phase0_validator.exit_epoch, withdrawable_epoch=phase0_validator.withdrawable_epoch, - next_custody_secret_to_reveal=get_custody_period_for_validator(validator_index, epoch), + next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(i), epoch), max_reveal_lateness=0, # TODO custody refactor. Outdated? - ) for validator_index, phase0_validator in enumerate(pre.validators) + ) for i, phase0_validator in enumerate(pre.validators) ), balances=pre.balances, # Randomness @@ -102,7 +102,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: exposed_derived_secrets=Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() ) - post.current_light_committee = get_light_client_committee(post, post.epoch) - post.next_light_committee = get_light_client_committee(post, post.epoch + 1) + post.current_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, post.epoch)) + post.next_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, post.epoch + 1)) return post ``` From ddd7cd29640c4216fb899a9b88d47c95bd7a58c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 22 Nov 2019 15:45:55 -0700 Subject: [PATCH 066/111] divide new and old containers out in phase 1 --- specs/core/1_beacon-chain.md | 170 ++++++++++++++++++----------------- 1 file changed, 88 insertions(+), 82 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 4fe49c92dc..6950d3517a 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -51,45 +51,14 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_COMMITTEE` | `192` | | | `DOMAIN_SHARD_PROPOSAL` | `193` | | -## Containers +## Updated containers -### `ShardBlockWrapper` - -_Wrapper for being broadcasted over the network._ - -```python -class ShardBlockWrapper(Container): - shard_parent_root: Hash - beacon_parent_root: Hash - slot: Slot - body: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] - signature: BLSSignature -``` - -### `ShardSignableHeader` - -```python -class ShardSignableHeader(Container): - shard_parent_root: Hash - beacon_parent_root: Hash - slot: Slot - body_root: Hash -``` +The following containers have updated definitions in Phase 1. -### `ShardState` +### Extended `AttestationData` ```python -class ShardState(Container): - slot: Slot - gasprice: Gwei - data: Hash - latest_block_root: Hash -``` - -### New `AttestationData` - -```python -class AttestationData(phase0.AttestationData): +class AttestationData(Container): slot: Slot index: CommitteeIndex # LMD GHOST vote @@ -103,23 +72,7 @@ class AttestationData(phase0.AttestationData): shard_transition_root: Hash ``` -### `ShardTransition` - -```python -class ShardTransition(Container): - # Starting from slot - start_slot: Slot - # Shard block lengths - shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] - # Shard data roots - shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] - # Intermediate shard states - shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] - # Proposer signature aggregate - proposer_signature_aggregate: BLSSignature -``` - -### New `Attestation` +### Extended `Attestation` ```python class Attestation(Container): @@ -129,32 +82,7 @@ class Attestation(Container): signature: BLSSignature ``` -### `AttestationAndCommittee` - -```python -class AttestationAndCommittee(Container): - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - attestation: Attestation -``` - -### `CompactCommittee` - -```python -class CompactCommittee(Container): - pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] -``` - -### `AttestationCustodyBitWrapper` - -```python -class AttestationCustodyBitWrapper(Container): - attestation_root: Hash - block_index: uint64 - bit: boolean -``` - -### New extended `PendingAttestation` +### Extended `PendingAttestation` ```python class PendingAttestation(Container): @@ -165,7 +93,7 @@ class PendingAttestation(Container): crosslink_success: boolean ``` -### New extended `Validator` +### Extended `Validator` ```python class Validator(Container): @@ -186,7 +114,7 @@ class Validator(Container): max_reveal_lateness: Epoch ``` -### New extended `BeaconBlockBody` +### Extended `BeaconBlockBody` ```python class BeaconBlockBody(Container): @@ -212,7 +140,7 @@ class BeaconBlockBody(Container): light_client_signature: BLSSignature ``` -### New extended `BeaconBlock` +### Extended `BeaconBlock` Note that the `body` has a new `BeaconBlockBody` definition. @@ -225,7 +153,7 @@ class BeaconBlock(Container): signature: BLSSignature ``` -### New extended `BeaconState` +### Extended `BeaconState` Note that aside from the new additions, `Validator` and `PendingAttestation` have new definitions. @@ -271,6 +199,84 @@ class BeaconState(Container): EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` +## New containers + +The following containers are new in Phase 1. + +### `ShardBlockWrapper` + +_Wrapper for being broadcasted over the network._ + +```python +class ShardBlockWrapper(Container): + shard_parent_root: Hash + beacon_parent_root: Hash + slot: Slot + body: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] + signature: BLSSignature +``` + +### `ShardSignableHeader` + +```python +class ShardSignableHeader(Container): + shard_parent_root: Hash + beacon_parent_root: Hash + slot: Slot + body_root: Hash +``` + +### `ShardState` + +```python +class ShardState(Container): + slot: Slot + gasprice: Gwei + data: Hash + latest_block_root: Hash +``` + +### `ShardTransition` + +```python +class ShardTransition(Container): + # Starting from slot + start_slot: Slot + # Shard block lengths + shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] + # Shard data roots + shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] + # Intermediate shard states + shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] + # Proposer signature aggregate + proposer_signature_aggregate: BLSSignature +``` + +### `AttestationAndCommittee` + +```python +class AttestationAndCommittee(Container): + committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + attestation: Attestation +``` + +### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + +### `AttestationCustodyBitWrapper` + +```python +class AttestationCustodyBitWrapper(Container): + attestation_root: Hash + block_index: uint64 + bit: boolean +``` + ## Helper functions ### Crypto From a1ac0d5a80df8d88dccae15a1b9fc2ac5aca4372 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 22 Nov 2019 16:26:36 -0700 Subject: [PATCH 067/111] fix get_shard_committee --- specs/core/1_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 6950d3517a..d4531d7d95 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -354,7 +354,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) - return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS) + return compute_committee(active_validator_indices, seed, shard, ACTIVE_SHARDS) ``` #### `get_shard_proposer_index` From f533fef1670d8827dcf92205c99239d86f6d1e00 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 22 Nov 2019 18:09:31 -0700 Subject: [PATCH 068/111] add missing attestation validations; cleanup process_attestations and fix minor bugs --- specs/core/1_beacon-chain.md | 169 ++++++++++++++++++++++------------- specs/core/1_phase1-fork.md | 2 +- 2 files changed, 110 insertions(+), 61 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index d4531d7d95..510ca8d5f0 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -415,6 +415,14 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: return Shard((attestation.data.index + get_start_shard(state, attestation.data.slot)) % ACTIVE_SHARDS) ``` +#### `get_next_slot_for_shard` + +```python +def get_next_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + return Slot(state.shard_transitions[shard].slot + 1) +``` + + #### `get_offset_slots` ```python @@ -422,7 +430,6 @@ def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] ``` - ### Predicates #### Updated `is_valid_indexed_attestation` @@ -507,17 +514,30 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ```python def validate_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data + assert data.index < get_committee_count_at_slot(state, data.slot) assert data.index < ACTIVE_SHARDS + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + + if attestation.data.target.epoch == get_current_epoch(state): + assert attestation.data.source == state.current_justified_checkpoint + else: + assert attestation.data.source == state.previous_justified_checkpoint + shard = get_shard(state, attestation) + shard_start_slot = get_next_slot_for_shard(state, shard) # Signature check assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Type 1: on-time attestations if attestation.custody_bits != []: # Correct slot - assert data.slot == state.slot + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Correct data root count - assert len(attestation.custody_bits) == len(get_offset_slots(state, state.shard_next_slots[shard])) + assert len(attestation.custody_bits) == len(get_offset_slots(state, shard_start_slot)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state)) # Type 2: delayed attestations @@ -531,7 +551,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: # Slot the attestation starts counting from - start_slot = state.shard_next_slots[shard] + start_slot = get_next_slot_for_shard(state, shard) # Correct data root count offset_slots = get_offset_slots(state, start_slot) @@ -543,7 +563,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr ) assert transition.start_slot == start_slot - # Reonstruct shard headers + # Reconstruct shard headers headers = [] proposers = [] shard_parent_root = state.shard_states[shard].latest_block_root @@ -582,79 +602,108 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr state.shard_states[shard].slot = state.slot - 1 ``` -###### `process_attestations` +###### `process_crosslink_for_shard` ```python -def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: - # Basic validation - for attestation in attestations: - validate_attestation(state, attestation) - # Process crosslinks +def process_crosslink_for_shard(state: BeaconState, + shard: Shard, + shard_transition: ShardTransition, + attestations: Sequence[Attestation]) -> Hash: + committee = get_beacon_committee(state, get_current_epoch(state), shard) online_indices = get_online_validator_indices(state) - winners = set() + + # Loop over all shard transition roots + shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) + for shard_transition_root in sorted(shard_transition_roots): + transition_attestations = [a for a in attestations if a.data.shard_transition_root == shard_transition_root] + transition_participants: Set[ValidatorIndex] = set() + for attestation in transition_attestations: + participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + transition_participants = transition_participants.union(participants) + + enough_online_stake = ( + get_total_balance(state, online_indices.intersection(transition_participants)) * 3 >= + get_total_balance(state, online_indices.intersection(committee)) * 2 + ) + # If not enough stake, try next transition root + if not enough_online_stake: + continue + + # Attestation <-> shard transition consistency + assert shard_transition_root == hash_tree_root(shard_transition) + assert ( + attestation.data.head_shard_root + == chunks_to_body_root(shard_transition.shard_data_roots[-1]) + ) + + # Apply transition + apply_shard_transition(state, shard, shard_transition) + # Apply proposer reward and cost + beacon_proposer_index = get_beacon_proposer_index(state) + estimated_attester_reward = sum([get_base_reward(state, attester) for attester in transition_participants]) + proposer_reward = Gwei(estimated_attester_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, beacon_proposer_index, proposer_reward) + states_slots_lengths = zip( + shard_transition.shard_states, + get_offset_slots(state, get_next_slot_for_shard(state, shard)), + shard_transition.shard_block_lengths + ) + for shard_state, slot, length in states_slots_lengths: + proposer_index = get_shard_proposer_index(state, shard, slot) + decrease_balance(state, proposer_index, shard_state.gasprice * length) + + # Return winning transition root + return shard_transition_root + + # No winning transition root, ensure empty and return empty root + assert shard_transition == ShardTransition() + return Hash() +``` + +###### `process_crosslinks` + +```python +def process_crosslinks(state: BeaconState, + block_body: BeaconBlockBody, + attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Hash]]: + winners: Set[Tuple[Shard, Hash]] = set() for shard in map(Shard, range(ACTIVE_SHARDS)): - success = False # All attestations in the block for this shard - this_shard_attestations = [ + shard_attestations = [ attestation for attestation in attestations if get_shard(state, attestation) == shard and attestation.data.slot == state.slot ] - # The committee for this shard - this_shard_committee = get_beacon_committee(state, get_current_epoch(state), shard) - # Loop over all shard transition roots - shard_transition_roots = set([a.data.shard_transition_root for a in this_shard_attestations]) - for shard_transition_root in sorted(shard_transition_roots): - all_participants: Set[ValidatorIndex] = set() - participating_attestations = [] - for attestation in this_shard_attestations: - participating_attestations.append(attestation) - if attestation.data.shard_transition_root == shard_transition_root: - participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - all_participants = all_participants.union(participants) - if ( - get_total_balance(state, online_indices.intersection(all_participants)) * 3 >= - get_total_balance(state, online_indices.intersection(this_shard_committee)) * 2 - and success is False - ): - # Attestation <-> shard transition consistency - assert shard_transition_root == hash_tree_root(block_body.shard_transition) - assert ( - attestation.data.head_shard_root - == chunks_to_body_root(block_body.shard_transition.shard_data_roots[-1]) - ) - # Apply transition - apply_shard_transition(state, shard, block_body.shard_transition) - # Apply proposer reward and cost - beacon_proposer_index = get_beacon_proposer_index(state) - estimated_attester_reward = sum([get_base_reward(state, attester) for attester in all_participants]) - proposer_reward = Gwei(estimated_attester_reward // PROPOSER_REWARD_QUOTIENT) - increase_balance(state, beacon_proposer_index, proposer_reward) - states_slots_lengths = zip( - block_body.shard_transition.shard_states, - get_offset_slots(state, state.shard_next_slots[get_shard(state, attestation)]), - block_body.shard_transition.shard_block_lengths - ) - for shard_state, slot, length in states_slots_lengths: - proposer_index = get_shard_proposer_index(state, shard, slot) - decrease_balance(state, proposer_index, shard_state.gasprice * length) - winners.add((shard, shard_transition_root)) - success = True - if not success: - assert block_body.shard_transitions[shard] == ShardTransition() + shard_transition = block_body.shard_transitions[shard] + winning_root = process_crosslink_for_shard(state, shard, shard_transition, shard_attestations) + if winning_root != Hash(): + winners.add((shard, winning_root)) + return winners +``` + +###### `process_attestations` + +```python +def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: + # Basic validation + for attestation in attestations: + validate_attestation(state, attestation) + + # Process crosslinks + winners = process_crosslinks(state, block_body, attestations) + + # Store pending attestations for epoch processing for attestation in attestations: - is_winning_transition = (get_shard(state, attestation), attestation.shard_transition_root) in winners + is_winning_transition = (get_shard(state, attestation), attestation.data.shard_transition_root) in winners pending_attestation = PendingAttestation( aggregation_bits=attestation.aggregation_bits, data=attestation.data, inclusion_delay=state.slot - attestation.data.slot, crosslink_success=is_winning_transition and attestation.data.slot == state.slot, - proposer_index=proposer_index + proposer_index=get_beacon_proposer_index(state), ) if attestation.data.target.epoch == get_current_epoch(state): - assert attestation.data.source == state.current_justified_checkpoint state.current_epoch_attestations.append(pending_attestation) else: - assert attestation.data.source == state.previous_justified_checkpoint state.previous_epoch_attestations.append(pending_attestation) ``` diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index a1e14ec354..1ec8acaa06 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -86,7 +86,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: # Phase 1 shard_states=List[ShardState, MAX_SHARDS]( ShardState( - slot=0, + slot=pre.slot, gasprice=INITIAL_GASPRICE, data=Root(), latest_block_root=Hash(), From 4bf5eb86c46b6899a45d06db502230901a338c29 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 22 Nov 2019 18:17:38 -0700 Subject: [PATCH 069/111] fix issues using indexed attestation and custody bit wrapper --- specs/core/1_beacon-chain.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 510ca8d5f0..300d45fdf7 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -272,7 +272,7 @@ class CompactCommittee(Container): ```python class AttestationCustodyBitWrapper(Container): - attestation_root: Hash + attestation_data_root: Hash block_index: uint64 bit: boolean ``` @@ -441,20 +441,20 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Attest """ Check if ``indexed_attestation`` has valid indices and signature. """ - # Verify aggregate signature all_pubkeys = [] all_message_hashes = [] - aggregation_bits = indexed_attestation.attestation.aggregation_bits + attestation = indexed_attestation.attestation + aggregation_bits = attestation.aggregation_bits assert len(aggregation_bits) == len(indexed_attestation.committee) - for i, custody_bits in enumerate(indexed_attestation.attestation.custody_bits): + for i, custody_bits in enumerate(attestation.custody_bits): assert len(custody_bits) == len(indexed_attestation.committee) for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): if abit: all_pubkeys.append(state.validators[participant].pubkey) # Note: only 2N distinct message hashes all_message_hashes.append(hash_tree_root( - AttestationCustodyBitWrapper(hash_tree_root(indexed_attestation.data), i, cbit) + AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit) )) else: assert not cbit @@ -462,8 +462,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Attest return bls_verify_multiple( pubkeys=all_pubkeys, message_hashes=all_message_hashes, - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch), + signature=attestation.signature, + domain=get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch), ) ``` From a32b0100ff338561b5f88425ab3eca8e1aaebf28 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 5 Dec 2019 12:36:48 -0700 Subject: [PATCH 070/111] rebase fixes --- scripts/build_spec.py | 4 +-- specs/core/1_beacon-chain.md | 60 ++++++++++++++++++------------------ specs/core/1_custody-game.md | 2 +- specs/core/1_fraud-proofs.md | 6 ++-- specs/core/1_phase1-fork.md | 4 +-- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 6dfc4e6b28..b88104140a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -165,10 +165,10 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', + 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', - 'bytes', 'byte', 'Bytes', 'BytesN' # to be removed after updating spec doc + 'bytes', 'byte', 'ByteList', 'ByteVector' ] diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 300d45fdf7..6837987b1c 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -43,7 +43,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | | +| `EMPTY_CHUNK_ROOT` | `hash_tree_root(ByteList[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | @@ -62,14 +62,14 @@ class AttestationData(Container): slot: Slot index: CommitteeIndex # LMD GHOST vote - beacon_block_root: Hash + beacon_block_root: Root # FFG vote source: Checkpoint target: Checkpoint # Current-slot shard block root - head_shard_root: Hash + head_shard_root: Root # Shard transition root - shard_transition_root: Hash + shard_transition_root: Root ``` ### Extended `Attestation` @@ -98,7 +98,7 @@ class PendingAttestation(Container): ```python class Validator(Container): pubkey: BLSPubkey - withdrawal_credentials: Hash # Commitment to pubkey for withdrawals + withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals effective_balance: Gwei # Balance at stake slashed: boolean # Status epochs @@ -147,8 +147,8 @@ Note that the `body` has a new `BeaconBlockBody` definition. ```python class BeaconBlock(Container): slot: Slot - parent_root: Hash - state_root: Hash + parent_root: Root + state_root: Root body: BeaconBlockBody signature: BLSSignature ``` @@ -165,9 +165,9 @@ class BeaconState(Container): fork: Fork # History latest_block_header: BeaconBlockHeader - block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash, HISTORICAL_ROOTS_LIMIT] + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] # Eth1 eth1_data: Eth1Data eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] @@ -176,7 +176,7 @@ class BeaconState(Container): validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] # Randomness - randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + randao_mixes: Vector[Root, EPOCHS_PER_HISTORICAL_VECTOR] # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Attestations @@ -189,7 +189,7 @@ class BeaconState(Container): finalized_checkpoint: Checkpoint # Phase 1 shard_states: List[ShardState, MAX_SHARDS] - online_countdown: Bytes[VALIDATOR_REGISTRY_LIMIT] + online_countdown: ByteList[VALIDATOR_REGISTRY_LIMIT] current_light_committee: CompactCommittee next_light_committee: CompactCommittee # Custody game @@ -209,10 +209,10 @@ _Wrapper for being broadcasted over the network._ ```python class ShardBlockWrapper(Container): - shard_parent_root: Hash - beacon_parent_root: Hash + shard_parent_root: Root + beacon_parent_root: Root slot: Slot - body: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] + body: ByteList[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] signature: BLSSignature ``` @@ -220,10 +220,10 @@ class ShardBlockWrapper(Container): ```python class ShardSignableHeader(Container): - shard_parent_root: Hash - beacon_parent_root: Hash + shard_parent_root: Root + beacon_parent_root: Root slot: Slot - body_root: Hash + body_root: Root ``` ### `ShardState` @@ -232,8 +232,8 @@ class ShardSignableHeader(Container): class ShardState(Container): slot: Slot gasprice: Gwei - data: Hash - latest_block_root: Hash + data: Bytes32 + latest_block_root: Root ``` ### `ShardTransition` @@ -245,7 +245,7 @@ class ShardTransition(Container): # Shard block lengths shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots - shard_data_roots: List[List[Hash, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] + shard_data_roots: List[List[Bytes32, MAX_SHARD_BLOCK_CHUNKS], MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate shard states shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Proposer signature aggregate @@ -272,7 +272,7 @@ class CompactCommittee(Container): ```python class AttestationCustodyBitWrapper(Container): - attestation_data_root: Hash + attestation_data_root: Root block_index: uint64 bit: boolean ``` @@ -319,8 +319,8 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid #### `chunks_to_body_root` ```python -def chunks_to_body_root(chunks: List[Hash, MAX_SHARD_BLOCK_CHUNKS]) -> Hash: - return hash_tree_root(Vector[Hash, MAX_SHARD_BLOCK_CHUNKS]( +def chunks_to_body_root(chunks: List[Bytes32, MAX_SHARD_BLOCK_CHUNKS]) -> Root: + return hash_tree_root(Vector[Bytes32, MAX_SHARD_BLOCK_CHUNKS]( chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) )) ``` @@ -543,7 +543,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Type 2: delayed attestations else: assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH - assert data.shard_transition_root == Hash() + assert data.shard_transition_root == Root() ``` ###### `apply_shard_transition` @@ -608,7 +608,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr def process_crosslink_for_shard(state: BeaconState, shard: Shard, shard_transition: ShardTransition, - attestations: Sequence[Attestation]) -> Hash: + attestations: Sequence[Attestation]) -> Root: committee = get_beacon_committee(state, get_current_epoch(state), shard) online_indices = get_online_validator_indices(state) @@ -657,7 +657,7 @@ def process_crosslink_for_shard(state: BeaconState, # No winning transition root, ensure empty and return empty root assert shard_transition == ShardTransition() - return Hash() + return Root() ``` ###### `process_crosslinks` @@ -665,8 +665,8 @@ def process_crosslink_for_shard(state: BeaconState, ```python def process_crosslinks(state: BeaconState, block_body: BeaconBlockBody, - attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Hash]]: - winners: Set[Tuple[Shard, Hash]] = set() + attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Root]]: + winners: Set[Tuple[Shard, Root]] = set() for shard in map(Shard, range(ACTIVE_SHARDS)): # All attestations in the block for this shard shard_attestations = [ @@ -675,7 +675,7 @@ def process_crosslinks(state: BeaconState, ] shard_transition = block_body.shard_transitions[shard] winning_root = process_crosslink_for_shard(state, shard, shard_transition, shard_attestations) - if winning_root != Hash(): + if winning_root != Root(): winners.add((shard, winning_root)) return winners ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 61710cd5f8..aa96b387f0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -100,7 +100,7 @@ class CustodySlashing(Container): whistleblower_index: ValidatorIndex shard_transition: ShardTransition attestation: Attestation - data: Bytes[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] + data: ByteList[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] signature: BLSSignature ``` diff --git a/specs/core/1_fraud-proofs.md b/specs/core/1_fraud-proofs.md index bd5cce117b..8c424d259b 100644 --- a/specs/core/1_fraud-proofs.md +++ b/specs/core/1_fraud-proofs.md @@ -34,10 +34,10 @@ The proof verifies that one of the two conditions is false: ```python def shard_state_transition(shard: Shard, slot: Slot, - pre_state: Hash, - previous_beacon_root: Hash, + pre_state: Root, + previous_beacon_root: Root, proposer_pubkey: BLSPubkey, - block_data: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Hash: + block_data: ByteVector[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]) -> Root: # We will add something more substantive in phase 2 return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) ``` diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index 1ec8acaa06..0b5e2085b0 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -89,10 +89,10 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: slot=pre.slot, gasprice=INITIAL_GASPRICE, data=Root(), - latest_block_root=Hash(), + latest_block_root=Root(), ) for i in range(ACTIVE_SHARDS) ), - online_countdown=Bytes[VALIDATOR_REGISTRY_LIMIT]( + online_countdown=ByteList[VALIDATOR_REGISTRY_LIMIT]( ONLINE_PERIOD for i in range(len(pre.validators)) ), current_light_committee=CompactCommittee(), # computed after state creation From e31f17f03700da5c5edf3a555c4951ee8ee1059f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 5 Dec 2019 13:49:52 -0700 Subject: [PATCH 071/111] modify phase 1 to use new signature (no signing_root) pattern --- scripts/build_spec.py | 5 +---- specs/core/1_beacon-chain.md | 13 +++++++++++-- specs/core/1_custody-game.md | 13 +++++++++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index b88104140a..b1a8b3485f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -51,10 +51,7 @@ field, ) -from eth2spec.utils.ssz.ssz_impl import ( - hash_tree_root, - is_zero, -) +from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( BasicValue, Elements, BaseBytes, BaseList, SSZType, Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector, Bits, diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 6837987b1c..3e88d01da2 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -128,9 +128,9 @@ class BeaconBlockBody(Container): attestations: List[Attestation, MAX_ATTESTATIONS] # Entry & exit deposits: List[Deposit, MAX_DEPOSITS] - voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game - custody_slashings: List[CustodySlashing, MAX_CUSTODY_SLASHINGS] + custody_slashings: List[SignedCustodySlashing, MAX_CUSTODY_SLASHINGS] custody_key_reveals: List[CustodyKeyReveal, MAX_CUSTODY_KEY_REVEALS] early_derived_secret_reveals: List[EarlyDerivedSecretReveal, MAX_EARLY_DERIVED_SECRET_REVEALS] # Shards @@ -150,6 +150,15 @@ class BeaconBlock(Container): parent_root: Root state_root: Root body: BeaconBlockBody +``` + +#### Extended `SignedBeaconBlock` + +Note that the `message` has a new `BeaconBlock` definition. + +```python +class SignedBeaconBlock(Container): + message: BeaconBlock signature: BLSSignature ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index aa96b387f0..60d63db03f 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -101,9 +101,17 @@ class CustodySlashing(Container): shard_transition: ShardTransition attestation: Attestation data: ByteList[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE] +``` + +#### `SignedCustodySlashing` + +```python +class SignedCustodySlashing(Container): + message: CustodySlashing signature: BLSSignature ``` + #### `CustodyKeyReveal` ```python @@ -347,7 +355,8 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived #### Custody Slashings ```python -def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashing) -> None: +def process_custody_slashing(state: BeaconState, signed_custody_slashing: SignedCustodySlashing) -> None: + custody_slashing = signed_custody_slashing.message attestation = custody_slashing.attestation # Any signed custody-slashing should result in at least one slashing. @@ -355,7 +364,7 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi malefactor = state.validators[custody_slashing.malefactor_index] whistleblower = state.validators[custody_slashing.whistleblower_index] domain = get_domain(state, DOMAIN_CUSTODY_BIT_SLASHING, get_current_epoch(state)) - assert bls_verify(whistleblower.pubkey, signing_root(custody_slashing), custody_slashing.signature, domain) + assert bls_verify(whistleblower.pubkey, hash_tree_root(custody_slashing), signed_custody_slashing.signature, domain) # Verify that the whistleblower is slashable assert is_slashable_validator(whistleblower, get_current_epoch(state)) # Verify that the claimed malefactor is slashable From ce406660031c227e532f9bebec96fa4821824067 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 5 Dec 2019 15:06:32 -0700 Subject: [PATCH 072/111] version pyspec module and test on one attesterslashing test --- scripts/build_spec.py | 6 +- specs/core/1_beacon-chain.md | 78 ++++++++++++++++--- .../eth2spec/test/helpers/attestations.py | 14 +++- .../test_process_attester_slashing.py | 6 +- 4 files changed, 87 insertions(+), 17 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index b1a8b3485f..f3a76817fc 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -112,6 +112,7 @@ def objects_to_spec(functions: Dict[str, str], constants: Dict[str, str], ssz_objects: Dict[str, str], imports: Dict[str, str], + version: str, ) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -137,6 +138,7 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) spec = ( imports + + '\n\n' + f"version = \'{version}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS + '\n\n' + constants_spec @@ -229,7 +231,7 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, for value in [fork_choice_spec, v_guide]: spec_objects = combine_spec_objects(spec_objects, value) dependency_order_spec(spec_objects) - spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS) + spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS, 'phase0') if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -258,7 +260,7 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, for value in all_spescs[1:]: spec_objects = combine_spec_objects(spec_objects, value) dependency_order_spec(spec_objects) - spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) + spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, 'phase1') if outfile is not None: with open(outfile, 'w') as out: out.write(spec) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 3e88d01da2..b0c0b5fbf2 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -93,6 +93,24 @@ class PendingAttestation(Container): crosslink_success: boolean ``` +### `IndexedAttestation` + +```python +class IndexedAttestation(Container): + committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + attestation: Attestation +``` + +#### Extended `AttesterSlashing` + +Note that the `attestation_1` and `attestation_2` have a new `IndexedAttestation` definition. + +```python +class AttesterSlashing(Container): + attestation_1: IndexedAttestation + attestation_2: IndexedAttestation +``` + ### Extended `Validator` ```python @@ -261,14 +279,6 @@ class ShardTransition(Container): proposer_signature_aggregate: BLSSignature ``` -### `AttestationAndCommittee` - -```python -class AttestationAndCommittee(Container): - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - attestation: Attestation -``` - ### `CompactCommittee` ```python @@ -390,9 +400,12 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque #### `get_indexed_attestation` ```python -def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> AttestationAndCommittee: +def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: committee = get_beacon_committee(beacon_state, attestation.data.slot, attestation.data.index) - return AttestationAndCommittee(committee, attestation) + return IndexedAttestation( + committee=committee, + attestation=attestation, + ) ``` #### `get_updated_gasprice` @@ -446,7 +459,7 @@ def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: Note that this replaces the Phase 0 `is_valid_indexed_attestation`. ```python -def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: AttestationAndCommittee) -> bool: +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ Check if ``indexed_attestation`` has valid indices and signature. """ @@ -467,7 +480,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Attest )) else: assert not cbit - + return bls_verify_multiple( pubkeys=all_pubkeys, message_hashes=all_message_hashes, @@ -716,6 +729,47 @@ def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attest state.previous_epoch_attestations.append(pending_attestation) ``` +##### New Attester slashing processing + +```python +def get_indices_from_committee( + committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE], + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]: + assert len(bits) == len(committee) + return List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]( + [validator_index for i, validator_index in enumerate(committee) if bits[i]] + ) +``` + +```python +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: + indexed_attestation_1 = attester_slashing.attestation_1 + indexed_attestation_2 = attester_slashing.attestation_2 + assert is_slashable_attestation_data( + indexed_attestation_1.attestation.data, + indexed_attestation_2.attestation.data, + ) + assert is_valid_indexed_attestation(state, indexed_attestation_1) + assert is_valid_indexed_attestation(state, indexed_attestation_2) + + indices_1 = get_indices_from_committee( + indexed_attestation_1.committee, + indexed_attestation_1.attestation.aggregation_bits, + ) + indices_2 = get_indices_from_committee( + indexed_attestation_2.committee, + indexed_attestation_2.attestation.aggregation_bits, + ) + + slashed_any = False + indices = set(indices_1).intersection(indices_2) + for index in sorted(indices): + if is_slashable_validator(state.validators[index], get_current_epoch(state)): + slash_validator(state, index) + slashed_any = True + assert slashed_any +``` + #### Shard transition false positives ```python diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 3ed54888f4..0c64a03165 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -82,8 +82,18 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List def sign_indexed_attestation(spec, state, indexed_attestation): - participants = indexed_attestation.attesting_indices - indexed_attestation.signature = sign_aggregate_attestation(spec, state, indexed_attestation.data, participants) + if spec.version == 'phase0': + participants = indexed_attestation.attesting_indices + data = indexed_attestation.data + indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) + else: + participants = spec.get_indices_from_committee( + indexed_attestation.committee, + indexed_attestation.attestation.aggregation_bits, + ) + data = indexed_attestation.attestation.data + indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) + def sign_attestation(spec, state, attestation): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 85e807ec00..9a227625ae 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -169,7 +169,11 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target.epoch += 1 + if spec.version == 'phase0': + attester_slashing.attestation_1.data.target.epoch += 1 + else: + attester_slashing.attestation_1.attestation.data.target.epoch += 1 + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) From 0a3cc8a84a6c834242d90a7cf4696ca7d5725598 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Jan 2020 07:29:23 -0700 Subject: [PATCH 073/111] malefactor_key -> malefactor_secret --- specs/core/1_custody-game.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 60d63db03f..c92af627bc 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -96,7 +96,7 @@ class CustodySlashing(Container): # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. data_index: uint64 malefactor_index: ValidatorIndex - malefactor_key: BLSSignature + malefactor_secret: BLSSignature whistleblower_index: ValidatorIndex shard_transition: ShardTransition attestation: Attestation @@ -393,7 +393,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed custody_slashing.malefactor_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - assert bls_verify(malefactor.pubkey, hash_tree_root(epoch_to_sign), custody_slashing.malefactor_key, domain) + assert bls_verify(malefactor.pubkey, hash_tree_root(epoch_to_sign), custody_slashing.malefactor_secret, domain) # Get the custody bit custody_bits = attestation.custody_bits[custody_slashing.data_index] @@ -401,7 +401,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] # Compute the custody bit - computed_custody_bit = compute_custody_bit(custody_slashing.malefactor_key, custody_slashing.data) + computed_custody_bit = compute_custody_bit(custody_slashing.malefactor_secret, custody_slashing.data) # Verify the claim if claimed_custody_bit != computed_custody_bit: From 13cdfa8edb748d2d108fcb8bafdcb7e73888b63b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 4 Jan 2020 18:33:15 +0100 Subject: [PATCH 074/111] apply configs in tests properly --- test_libs/pyspec/eth2spec/config/apply_config.py | 3 +++ test_libs/pyspec/eth2spec/test/conftest.py | 3 +++ test_libs/pyspec/eth2spec/test/context.py | 6 ++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/config/apply_config.py b/test_libs/pyspec/eth2spec/config/apply_config.py index 768abba648..2f0ce59021 100644 --- a/test_libs/pyspec/eth2spec/config/apply_config.py +++ b/test_libs/pyspec/eth2spec/config/apply_config.py @@ -5,6 +5,7 @@ # Access to overwrite spec constants based on configuration +# This is called by the spec module after declaring its globals, and applies the loaded presets. def apply_constants_preset(spec_globals: Dict[str, Any]) -> None: global presets for k, v in presets.items(): @@ -14,6 +15,8 @@ def apply_constants_preset(spec_globals: Dict[str, Any]) -> None: spec_globals[k] = v +# Load presets from a file. This does not apply the presets. +# To apply the presets, reload the spec module (it will re-initialize with the presets taken from here). def load_presets(configs_path, config_name): global presets presets = loader.load_presets(configs_path, config_name) diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py index a0a200f0bb..35ffb3cb88 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -1,4 +1,5 @@ from eth2spec.config import apply_config +from eth2spec.test.context import reload_specs # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. @@ -33,3 +34,5 @@ def pytest_addoption(parser): def config(request): config_name = request.config.getoption("--config") apply_config.load_presets('../../configs/', config_name) + # now that the presets are loaded, reload the specs to apply them + reload_specs() diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 0fb46aa50b..195d1e5fa5 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -10,8 +10,10 @@ from importlib import reload -reload(spec_phase0) -reload(spec_phase1) + +def reload_specs(): + reload(spec_phase0) + reload(spec_phase1) def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], From 018927def0b1e0cd27bfb42a3998a9031bad9249 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 20:11:55 +0100 Subject: [PATCH 075/111] generalize previous-slot function --- specs/core/1_beacon-chain.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index e44dcfcb7d..a4b79df386 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -29,11 +29,11 @@ - [Crypto](#crypto) - [`bls_verify_multiple`](#bls_verify_multiple) - [Misc](#misc-1) + - [`get_previous_slot`](#get_previous_slot) - [`pack_compact_validator`](#pack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`chunks_to_body_root`](#chunks_to_body_root) - [Beacon state accessors](#beacon-state-accessors) - - [`get_previous_slot`](#get_previous_slot) - [`get_online_validator_indices`](#get_online_validator_indices) - [`get_shard_committee`](#get_shard_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) @@ -98,7 +98,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | Name | Value | Unit | Duration | | - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ACTIVE_SHARDS` | `2**6` (= 64) | | `ONLINE_PERIOD` | `Epoch(2**3)` (= 8) | epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | @@ -372,6 +371,16 @@ class AttestationCustodyBitWrapper(Container): ### Misc +#### `get_previous_slot` + +```python +def get_previous_slot(slot: Slot) -> Slot: + if slot > 0: + return Slot(slot - 1) + else: + return Slot(0) +``` + #### `pack_compact_validator` ```python @@ -411,16 +420,6 @@ def chunks_to_body_root(chunks: List[Bytes32, MAX_SHARD_BLOCK_CHUNKS]) -> Root: ### Beacon state accessors -#### `get_previous_slot` - -```python -def get_previous_slot(state: BeaconState) -> Slot: - if state.slot > 0: - return Slot(state.slot - 1) - else: - return Slot(0) -``` - #### `get_online_validator_indices` ```python @@ -626,7 +625,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Correct data root count assert len(attestation.custody_bits) == len(get_offset_slots(state, shard_start_slot)) # Correct parent block root - assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state)) + assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) # Type 2: delayed attestations else: assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH @@ -658,7 +657,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr if any(transition.shard_data_roots): headers.append(ShardSignableHeader( shard_parent_root=shard_parent_root, - parent_hash=get_block_root_at_slot(state, get_previous_slot(state)), + parent_hash=get_block_root_at_slot(state, get_previous_slot(state.slot)), slot=offset_slots[i], body_root=chunks_to_body_root(transition.shard_data_roots[i]) )) @@ -863,7 +862,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB assert bls_verify( pubkey=bls_aggregate_pubkeys(signer_keys), - message_hash=get_block_root_at_slot(state, get_previous_slot(state)), + message_hash=get_block_root_at_slot(state, get_previous_slot(state.slot)), signature=block_body.light_client_signature, domain=DOMAIN_LIGHT_CLIENT ) From a8276f683ef31c00ef8db897859783b7c660cb9b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 20:20:20 +0100 Subject: [PATCH 076/111] don't hardcode shard count, bad example, need upgradeability --- specs/core/1_beacon-chain.md | 22 ++++++++++++++++------ specs/core/1_phase1-fork.md | 3 ++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index a4b79df386..118689cf01 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -34,6 +34,7 @@ - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`chunks_to_body_root`](#chunks_to_body_root) - [Beacon state accessors](#beacon-state-accessors) + - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) - [`get_shard_committee`](#get_shard_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) @@ -420,6 +421,13 @@ def chunks_to_body_root(chunks: List[Bytes32, MAX_SHARD_BLOCK_CHUNKS]) -> Root: ### Beacon state accessors +#### `get_active_shard_count` + +```python +def get_active_shard_count(state: BeaconState) -> uint64: + return len(state.shard_states) # May adapt in the future, or change over time. +``` + #### `get_online_validator_indices` ```python @@ -437,7 +445,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) - return compute_committee(active_validator_indices, seed, shard, ACTIVE_SHARDS) + return compute_committee(active_validator_indices, seed, shard, get_active_shard_count(beacon_state)) ``` #### `get_shard_proposer_index` @@ -458,7 +466,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) - return compute_committee(active_validator_indices, seed, 0, ACTIVE_SHARDS)[:TARGET_COMMITTEE_SIZE] + active_shards = get_active_shard_count(beacon_state) + return compute_committee(active_validator_indices, seed, 0, active_shards)[:TARGET_COMMITTEE_SIZE] ``` #### `get_indexed_attestation` @@ -498,7 +507,8 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: - return Shard((attestation.data.index + get_start_shard(state, attestation.data.slot)) % ACTIVE_SHARDS) + active_shards = get_active_shard_count(state) + return Shard((attestation.data.index + get_start_shard(state, attestation.data.slot)) % active_shards) ``` #### `get_next_slot_for_shard` @@ -601,7 +611,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: def validate_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data assert data.index < get_committee_count_at_slot(state, data.slot) - assert data.index < ACTIVE_SHARDS + assert data.index < get_active_shard_count(state) assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH @@ -753,7 +763,7 @@ def process_crosslinks(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Root]]: winners: Set[Tuple[Shard, Root]] = set() - for shard in map(Shard, range(ACTIVE_SHARDS)): + for shard in range(get_active_shard_count(state)): # All attestations in the block for this shard shard_attestations = [ attestation for attestation in attestations @@ -839,7 +849,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla ```python def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it - for shard in range(ACTIVE_SHARDS): + for shard in range(get_active_shard_count(state)): if state.shard_states[shard].slot != state.slot - 1: assert block_body.shard_transition[shard] == ShardTransition() ``` diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index 6d87a30b6c..1b3fddcd8d 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -35,6 +35,7 @@ TODO: very unstable/experimental. PLACEHOLDER. | Name | Value | Unit | | - | - | - | | `PHASE_1_FORK_VERSION` | `0x00000001` | `Version` | +| `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | `uint64` | | `INITIAL_GASPRICE` | `10` | `Gwei` | ## Fork to Phase 1 @@ -104,7 +105,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: gasprice=INITIAL_GASPRICE, data=Root(), latest_block_root=Root(), - ) for i in range(ACTIVE_SHARDS) + ) for i in range(INITIAL_ACTIVE_SHARDS) ), online_countdown=ByteList[VALIDATOR_REGISTRY_LIMIT]( ONLINE_PERIOD for i in range(len(pre.validators)) From 11011f2544336327aecebbc891ebc01fc0a482b7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:37:05 +0100 Subject: [PATCH 077/111] test context can handle multiple phases better now --- test_libs/pyspec/eth2spec/test/context.py | 88 +++++++++++++++++------ 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 195d1e5fa5..650f1ec818 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,7 +6,7 @@ from .utils import vector_test, with_meta_tags -from typing import Any, Callable, Sequence +from typing import Any, Callable, Sequence, TypedDict, Protocol from importlib import reload @@ -16,21 +16,48 @@ def reload_specs(): reload(spec_phase1) +# Some of the Spec module functionality is exposed here to deal with phase-specific changes. + +# TODO: currently phases are defined as python modules. +# It would be better if they would be more well-defined interfaces for stronger typing. +class Spec(Protocol): + version: str + + +class Phase0(Spec): + ... + + +class Phase1(Spec): + def upgrade_to_phase1(self, state: spec_phase0.BeaconState) -> spec_phase1.BeaconState: ... + + +# add transfer, bridge, etc. as the spec evolves +class SpecForks(TypedDict, total=False): + phase0: Phase0 + phase1: Phase1 + + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): def deco(fn): - def entry(*args, **kw): + def entry(*args, spec: Spec, phases: SpecForks, **kw): try: - spec = kw['spec'] - - balances = balances_fn(spec) - activation_threshold = threshold_fn(spec) - - kw['state'] = create_genesis_state(spec=spec, validator_balances=balances, - activation_threshold=activation_threshold) + p0 = phases["phase0"] + balances = balances_fn(p0) + activation_threshold = threshold_fn(p0) + + state = create_genesis_state(spec=p0, validator_balances=balances, + activation_threshold=activation_threshold) + if spec.version == 'phase1': + # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. + # Decide based on performance/consistency results later. + state = phases["phase1"].upgrade_to_phase1(state) + + kw['state'] = state except KeyError: raise TypeError('Spec decorator must come within state decorator to inject spec into state.') - return fn(*args, **kw) + return fn(*args, spec=spec, phases=phases, **kw) return entry return deco @@ -76,6 +103,19 @@ def misc_balances(spec): return [spec.MAX_EFFECTIVE_BALANCE] * num_validators + [spec.MIN_DEPOSIT_AMOUNT] * num_misc_validators +def single_phase(fn): + """ + Decorator that filters out the phases data. + most state tests only focus on behavior of a single phase (the "spec"). + This decorator is applied as part of spec_state_test(fn). + """ + def entry(*args, **kw): + if 'phases' in kw: + kw.pop('phases') + fn(*args, **kw) + return entry + + # BLS is turned off by default *for performance purposes during TESTING*. # The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). # - Some tests are marked as BLS-requiring, and ignore this setting. @@ -95,9 +135,9 @@ def spec_test(fn): return vector_test()(bls_switch(fn)) -# shorthand for decorating @spectest() @with_state +# shorthand for decorating @spectest() @with_state @single_phase def spec_state_test(fn): - return spec_test(with_state(fn)) + return spec_test(with_state(single_phase(fn))) def expect_assertion_error(fn): @@ -176,15 +216,12 @@ def decorator(fn): return decorator -def with_phases(phases): +def with_phases(phases, other_phases=None): """ - Decorator factory that returns a decorator that runs a test for the appropriate phases + Decorator factory that returns a decorator that runs a test for the appropriate phases. + Additional phases that do not initially run, but are made available through the test, are optional. """ def decorator(fn): - def run_with_spec_version(spec, *args, **kw): - kw['spec'] = spec - return fn(*args, **kw) - def wrapper(*args, **kw): run_phases = phases @@ -195,10 +232,21 @@ def wrapper(*args, **kw): return run_phases = [phase] + available_phases = set(run_phases) + if other_phases is not None: + available_phases += set(other_phases) + + phase_dir = {} + if 'phase0' in available_phases: + phase_dir['phase0'] = spec_phase0 + if 'phase1' in available_phases: + phase_dir['phase1'] = spec_phase1 + + # return is ignored whenever multiple phases are ran. If if 'phase0' in run_phases: - ret = run_with_spec_version(spec_phase0, *args, **kw) + ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw) if 'phase1' in run_phases: - ret = run_with_spec_version(spec_phase1, *args, **kw) + ret = fn(spec=spec_phase1, phases=phase_dir, *args, **kw) return ret return wrapper return decorator From 2e6090fcd8d47ddbb18eccdb96bb89002270fadf Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:37:42 +0100 Subject: [PATCH 078/111] bugfixes for experimental phase0 -> phase1 fork py code --- specs/core/1_phase1-fork.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index 1b3fddcd8d..97ad3252bb 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -30,13 +30,13 @@ This document describes the process of moving from Phase 0 to Phase 1 of Ethereu ## Configuration -TODO: very unstable/experimental. PLACEHOLDER. +Warning: this configuration is not definitive. -| Name | Value | Unit | +| Name | Value | | - | - | - | -| `PHASE_1_FORK_VERSION` | `0x00000001` | `Version` | -| `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | `uint64` | -| `INITIAL_GASPRICE` | `10` | `Gwei` | +| `PHASE_1_FORK_VERSION` | `Version('0x00000001')` | +| `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | +| `INITIAL_GASPRICE` | `Gwei(10)` | ## Fork to Phase 1 @@ -55,7 +55,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: genesis_time=pre.genesis_time, slot=pre.slot, fork=Fork( - previous_version=pre.current_version, + previous_version=pre.fork.current_version, current_version=PHASE_1_FORK_VERSION, epoch=epoch, ), @@ -114,10 +114,10 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: next_light_committee=CompactCommittee(), # Custody game custody_challenge_index=0, - exposed_derived_secrets=Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], - EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]() + # exposed_derived_secrets will fully default to zeroes ) - post.current_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, post.epoch)) - post.next_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, post.epoch + 1)) + epoch = get_current_epoch(post) + post.current_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, epoch)) + post.next_light_committee = committee_to_compact_committee(post, get_light_client_committee(post, epoch + 1)) return post ``` From 3995bd633c42201bf5656ab6190169118d35dd19 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:38:06 +0100 Subject: [PATCH 079/111] coerce container types, since classes do not match between phases --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 4ddff1b5e3..823a4a8d52 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -119,6 +119,8 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): return typ(v) elif isinstance(v, GeneratorType): return typ(v) + elif issubclass(typ, Container) and not isinstance(v, typ): + return typ(**{field_name: getattr(v, field_name) for field_name in typ.get_field_names()}) # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. if strict and not isinstance(v, typ): @@ -192,7 +194,7 @@ def get_fields(cls) -> Dict[str, SSZType]: return dict(cls.__annotations__) @classmethod - def get_field_names(cls) -> Iterable[SSZType]: + def get_field_names(cls) -> Iterable[str]: if not hasattr(cls, '__annotations__'): # no container fields return () return list(cls.__annotations__.keys()) From 752eeecaf4f065a87c9079ec6dfd0dc48419d6d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:44:20 +0100 Subject: [PATCH 080/111] old challenge based custody game tests are not relevant anymore --- .../test_process_bit_challenge.py | 350 ------------------ 1 file changed, 350 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py deleted file mode 100644 index ae6ff258c5..0000000000 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py +++ /dev/null @@ -1,350 +0,0 @@ -from eth2spec.test.helpers.custody import ( - get_valid_bit_challenge, - get_valid_custody_response, - get_custody_test_vector, - get_custody_merkle_root -) -from eth2spec.test.helpers.attestations import ( - get_valid_attestation, -) -from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.context import ( - with_all_phases_except, - spec_state_test, - expect_assertion_error, -) -from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing - - -def run_bit_challenge_processing(spec, state, custody_bit_challenge, valid=True): - """ - Run ``process_bit_challenge``, yielding: - - pre-state ('pre') - - CustodyBitChallenge ('custody_bit_challenge') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - yield 'pre', state - yield 'custody_bit_challenge', custody_bit_challenge - - if not valid: - expect_assertion_error(lambda: spec.process_bit_challenge(state, custody_bit_challenge)) - yield 'post', None - return - - spec.process_bit_challenge(state, custody_bit_challenge) - - assert state.custody_bit_challenge_records[state.custody_challenge_index - 1].chunk_bits_merkle_root == \ - hash_tree_root(custody_bit_challenge.chunk_bits) - assert state.custody_bit_challenge_records[state.custody_challenge_index - 1].challenger_index == \ - custody_bit_challenge.challenger_index - assert state.custody_bit_challenge_records[state.custody_challenge_index - 1].responder_index == \ - custody_bit_challenge.responder_index - - yield 'post', state - - -def run_custody_response_processing(spec, state, custody_response, valid=True): - """ - Run ``process_bit_challenge_response``, yielding: - - pre-state ('pre') - - CustodyResponse ('custody_response') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - yield 'pre', state - yield 'custody_response', custody_response - - if not valid: - expect_assertion_error(lambda: spec.process_custody_response(state, custody_response)) - yield 'post', None - return - - # TODO: Add capability to also process chunk challenges, not only bit challenges - challenge = state.custody_bit_challenge_records[custody_response.challenge_index] - pre_slashed_balance = get_balance(state, challenge.challenger_index) - - spec.process_custody_response(state, custody_response) - - slashed_validator = state.validators[challenge.challenger_index] - - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - assert get_balance(state, challenge.challenger_index) < pre_slashed_balance - yield 'post', state - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_challenge_appended(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_multiple_epochs_custody(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 3 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_many_epochs_custody(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 100 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_off_chain_attestation(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_invalid_custody_bit_challenge(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=True) - - yield from run_bit_challenge_processing(spec, state, challenge, valid=False) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_max_reveal_lateness_1(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - responder_index = challenge.responder_index - target_epoch = attestation.data.target.epoch - - state.validators[responder_index].max_reveal_lateness = 3 - - latest_reveal_epoch = spec.get_randao_epoch_for_custody_period( - spec.get_custody_period_for_validator(state, responder_index, target_epoch), - responder_index - ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - - while spec.get_current_epoch(state) < latest_reveal_epoch - 2: - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_max_reveal_lateness_2(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - responder_index = challenge.responder_index - - state.validators[responder_index].max_reveal_lateness = 3 - - for i in range(spec.get_randao_epoch_for_custody_period( - spec.get_custody_period_for_validator(state, responder_index), - responder_index - ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - 1): - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield from run_bit_challenge_processing(spec, state, challenge, False) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_challenge_index - 1 - - custody_response = get_valid_custody_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_response_processing(spec, state, custody_response) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response_multiple_epochs(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 3 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_challenge_index - 1 - - custody_response = get_valid_custody_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_response_processing(spec, state, custody_response) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response_many_epochs(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 100 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_challenge_index - 1 - - custody_response = get_valid_custody_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_response_processing(spec, state, custody_response) From bf86a716ee6b4ce1d01011dbafad943f0b2083dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:51:02 +0100 Subject: [PATCH 081/111] force-add in phase0 in test context, since state creation is not independent yet --- test_libs/pyspec/eth2spec/test/context.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 650f1ec818..1498367878 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -236,6 +236,10 @@ def wrapper(*args, **kw): if other_phases is not None: available_phases += set(other_phases) + # TODO: test state is dependent on phase0 but is immediately transitioned to phase1. + # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 + available_phases.add('phase0') + phase_dir = {} if 'phase0' in available_phases: phase_dir['phase0'] = spec_phase0 From 2f3919c176a3228843e163f4ccc799910e37f17e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 Jan 2020 23:51:24 +0100 Subject: [PATCH 082/111] bugfix: pass block body to sub processing functions --- specs/core/1_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 118689cf01..f7f1335891 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -571,8 +571,8 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - verify_shard_transition_false_positives(state, block) - process_light_client_signatures(state, block) + verify_shard_transition_false_positives(state, block.body) + process_light_client_signatures(state, block.body) process_operations(state, block.body) ``` From 0cf866e7993e86a12a3c32fdedd447a3ad922419 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:06:57 +0100 Subject: [PATCH 083/111] fix bug, test should run properly, forward generator in return --- test_libs/pyspec/eth2spec/test/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 1498367878..0add9f6382 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -112,7 +112,7 @@ def single_phase(fn): def entry(*args, **kw): if 'phases' in kw: kw.pop('phases') - fn(*args, **kw) + return fn(*args, **kw) return entry From 6b872da3ecc92747c6af28de3eda34bbb131b27e Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:17:48 +0100 Subject: [PATCH 084/111] light client bitfield is vector now, small size anyway, better separation of committee complexity from data init --- specs/core/1_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index f7f1335891..384c75f51f 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -219,7 +219,7 @@ class BeaconBlockBody(Container): # Shards shard_transitions: Vector[ShardTransition, MAX_SHARDS] # Light clients - light_client_signature_bitfield: Bitlist[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_signature_bitfield: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] light_client_signature: BLSSignature ``` @@ -862,11 +862,11 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB assert len(block_body.light_client_signature_bitfield) == len(committee) total_reward = Gwei(0) signer_keys = [] - for i, participant_bit in enumerate(block_body.light_client_signature_bitfield): - if participant_bit: - signer_keys.append(state.validators[committee[i]].pubkey) - increase_balance(state, committee[i], get_base_reward(state, committee[i])) - total_reward += get_base_reward(state, committee[i]) + for bit_index, participant_index in enumerate(committee): + if block_body.light_client_signature_bitfield[bit_index]: + signer_keys.append(state.validators[participant_index].pubkey) + increase_balance(state, participant_index, get_base_reward(state, participant_index)) + total_reward += get_base_reward(state, participant_index) increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) From ce84ddd20763bb46e8d40bb856f96375eef09c4c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:24:06 +0100 Subject: [PATCH 085/111] initial update of phase1 section of configs --- configs/mainnet.yaml | 5 +++++ configs/minimal.yaml | 8 ++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 99bba1d4e7..078b2cced7 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -146,3 +146,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 + + +# Phase 1 +# --------------------------------------------------------------- +INITIAL_ACTIVE_SHARDS: 64 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 4aacf17e19..90af6605f2 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -153,9 +153,5 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 # Phase 1 # --------------------------------------------------------------- -SHARD_SLOTS_PER_BEACON_SLOT: 2 -EPOCHS_PER_SHARD_PERIOD: 4 -# PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2 -PHASE_1_FORK_EPOCH: 8 -# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SLOTS_PER_EPOCH -PHASE_1_FORK_SLOT: 64 +# [customized] reduced for testing +INITIAL_ACTIVE_SHARDS: 4 From 9c07e26a4d8693b32c13d050aba89b7314551210 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:25:18 +0100 Subject: [PATCH 086/111] length assert not applicable anymore --- specs/core/1_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 384c75f51f..af7e6c299e 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -859,7 +859,6 @@ def verify_shard_transition_false_positives(state: BeaconState, block_body: Beac ```python def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockBody) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) - assert len(block_body.light_client_signature_bitfield) == len(committee) total_reward = Gwei(0) signer_keys = [] for bit_index, participant_index in enumerate(committee): From 68d93f945c6a10af3eb96743e6f7ddb3eef6e2d7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:44:22 +0100 Subject: [PATCH 087/111] online countdown fixes --- specs/core/1_beacon-chain.md | 9 +++++---- specs/core/1_phase1-fork.md | 4 +--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index af7e6c299e..0e66e2472e 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -88,6 +88,7 @@ We define the following Python custom types for type hinting and readability: | Name | SSZ equivalent | Description | | - | - | - | | `Shard` | `uint64` | a shard number | +| `OnlineEpochs` | `uint8` | online countdown epochs | ## Configuration @@ -99,7 +100,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | Name | Value | Unit | Duration | | - | - | - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ONLINE_PERIOD` | `Epoch(2**3)` (= 8) | epochs | ~51 min | +| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | @@ -281,7 +282,7 @@ class BeaconState(Container): finalized_checkpoint: Checkpoint # Phase 1 shard_states: List[ShardState, MAX_SHARDS] - online_countdown: ByteList[VALIDATOR_REGISTRY_LIMIT] + online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee next_light_committee: CompactCommittee # Custody game @@ -515,7 +516,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_next_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: - return Slot(state.shard_transitions[shard].slot + 1) + return Slot(state.shard_states[shard].slot + 1) ``` @@ -851,7 +852,7 @@ def verify_shard_transition_false_positives(state: BeaconState, block_body: Beac # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(get_active_shard_count(state)): if state.shard_states[shard].slot != state.slot - 1: - assert block_body.shard_transition[shard] == ShardTransition() + assert block_body.shard_transitions[shard] == ShardTransition() ``` #### Light client processing diff --git a/specs/core/1_phase1-fork.md b/specs/core/1_phase1-fork.md index 97ad3252bb..fdd97a24ec 100644 --- a/specs/core/1_phase1-fork.md +++ b/specs/core/1_phase1-fork.md @@ -107,9 +107,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: latest_block_root=Root(), ) for i in range(INITIAL_ACTIVE_SHARDS) ), - online_countdown=ByteList[VALIDATOR_REGISTRY_LIMIT]( - ONLINE_PERIOD for i in range(len(pre.validators)) - ), + online_countdown=[ONLINE_PERIOD] * len(pre.validators), # all online current_light_committee=CompactCommittee(), # computed after state creation next_light_committee=CompactCommittee(), # Custody game From ba10046cc1eab2e5baa7e66fcc31bcbf18f665c9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:49:32 +0100 Subject: [PATCH 088/111] minor lint fixes --- test_libs/pyspec/eth2spec/test/context.py | 3 ++- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 0add9f6382..6134243df0 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -29,7 +29,8 @@ class Phase0(Spec): class Phase1(Spec): - def upgrade_to_phase1(self, state: spec_phase0.BeaconState) -> spec_phase1.BeaconState: ... + def upgrade_to_phase1(self, state: spec_phase0.BeaconState) -> spec_phase1.BeaconState: + ... # add transfer, bridge, etc. as the spec evolves diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 0c64a03165..27014a3471 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -95,7 +95,6 @@ def sign_indexed_attestation(spec, state, indexed_attestation): indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) - def sign_attestation(spec, state, attestation): participants = spec.get_attesting_indices( state, From cd7a2149f9da0aa388722f2cd0f4647606c705fc Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 00:57:36 +0100 Subject: [PATCH 089/111] fix genesis tests: only run on single phase, ignore phases extra data --- .../pyspec/eth2spec/test/genesis/test_initialization.py | 4 +++- test_libs/pyspec/eth2spec/test/genesis/test_validity.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py index 462065bb93..65b9a41e72 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_test, with_phases +from eth2spec.test.context import spec_test, with_phases, single_phase from eth2spec.test.helpers.deposits import ( prepare_genesis_deposits, ) @@ -6,6 +6,7 @@ @with_phases(['phase0']) @spec_test +@single_phase def test_initialize_beacon_state_from_eth1(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT deposits, deposit_root, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) @@ -33,6 +34,7 @@ def test_initialize_beacon_state_from_eth1(spec): @with_phases(['phase0']) @spec_test +@single_phase def test_initialize_beacon_state_some_small_balances(spec): main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT main_deposits, _, deposit_data_list = prepare_genesis_deposits(spec, main_deposit_count, diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_validity.py b/test_libs/pyspec/eth2spec/test/genesis/test_validity.py index a003938e7e..a90b4a6956 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_validity.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_validity.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_test, with_phases +from eth2spec.test.context import spec_test, with_phases, single_phase from eth2spec.test.helpers.deposits import ( prepare_genesis_deposits, ) @@ -27,6 +27,7 @@ def run_is_valid_genesis_state(spec, state, valid=True): @with_phases(['phase0']) @spec_test +@single_phase def test_is_valid_genesis_state_true(spec): state = create_valid_beacon_state(spec) @@ -35,6 +36,7 @@ def test_is_valid_genesis_state_true(spec): @with_phases(['phase0']) @spec_test +@single_phase def test_is_valid_genesis_state_false_invalid_timestamp(spec): state = create_valid_beacon_state(spec) state.genesis_time = spec.MIN_GENESIS_TIME - 1 @@ -44,6 +46,7 @@ def test_is_valid_genesis_state_false_invalid_timestamp(spec): @with_phases(['phase0']) @spec_test +@single_phase def test_is_valid_genesis_state_true_more_balance(spec): state = create_valid_beacon_state(spec) state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE + 1 @@ -63,6 +66,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): @with_phases(['phase0']) @spec_test +@single_phase def test_is_valid_genesis_state_true_one_more_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1 deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) @@ -76,6 +80,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): @with_phases(['phase0']) @spec_test +@single_phase def test_is_valid_genesis_state_false_not_enough_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) From 9dd8d1d880d0a7c476fd674cbfad13ae9021faa7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 01:18:52 +0100 Subject: [PATCH 090/111] add two missing decorators --- .../test/phase_0/block_processing/test_process_attestation.py | 3 ++- .../epoch_processing/test_process_rewards_and_penalties.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index d48386fd49..7937614a46 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -6,7 +6,7 @@ spec_test, low_balances, with_custom_state, -) + single_phase) from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_aggregate_attestation, @@ -66,6 +66,7 @@ def test_success(spec, state): @with_all_phases @spec_test @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@single_phase def test_success_multi_proposer_index_iterations(spec, state): state.slot += spec.SLOTS_PER_EPOCH * 2 attestation = get_valid_attestation(spec, state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index b4fc46b7db..dd3dae50aa 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.test.context import spec_state_test, with_all_phases, spec_test, \ - misc_balances, with_custom_state, default_activation_threshold + misc_balances, with_custom_state, default_activation_threshold, single_phase from eth2spec.test.helpers.state import ( next_epoch, next_slot, @@ -96,6 +96,7 @@ def test_full_attestations(spec, state): @with_all_phases @spec_test @with_custom_state(balances_fn=misc_balances, threshold_fn=default_activation_threshold) +@single_phase def test_full_attestations_misc_balances(spec, state): attestations = prepare_state_with_full_attestations(spec, state) From f810e6b9c2b2eeefe4b37dd9bdf08cf731646bbb Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 9 Jan 2020 15:16:46 +0100 Subject: [PATCH 091/111] update to python v3.8 --- .circleci/config.yml | 22 +++++++++++----------- test_libs/pyspec/setup.py | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3870d9227..afa5b82d9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,32 +35,32 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v5-pyspec + venv_name: v6-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v5-pyspec + venv_name: v6-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v8-deposit-contract + venv_name: v9-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v8-deposit-contract + venv_name: v9-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: checkout_specs: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: # Restore git repo at point close to target branch/revision, to speed up checkout @@ -80,7 +80,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +92,7 @@ jobs: - save_pyspec_cached_venv test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - restore_cache: @@ -114,7 +114,7 @@ jobs: command: sudo npm install -g doctoc && make check_toc codespell: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - checkout @@ -123,7 +123,7 @@ jobs: command: pip install codespell --user && make codespell lint: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - restore_cache: @@ -134,7 +134,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - restore_cache: @@ -146,7 +146,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 working_directory: ~/specs-repo steps: - restore_cache: diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 5be0db7f82..204187efc0 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -3,6 +3,7 @@ setup( name='pyspec', packages=find_packages(), + python_requires=">=3.8, <4", tests_require=["pytest"], install_requires=[ "eth-utils>=1.3.0,<2", From 68ff136b5de5213093f0c9c4e8057eaaf8cceea5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 10 Jan 2020 00:00:10 +0100 Subject: [PATCH 092/111] warn about broken attestation validation, fix most attester slashings tests --- specs/core/1_beacon-chain.md | 13 ++-- specs/core/1_custody-game.md | 4 +- .../test/helpers/attester_slashings.py | 37 ++++++++++++ .../test_process_attester_slashing.py | 59 +++++++++++-------- .../eth2spec/test/sanity/test_blocks.py | 4 +- 5 files changed, 82 insertions(+), 35 deletions(-) diff --git a/specs/core/1_beacon-chain.md b/specs/core/1_beacon-chain.md index 23748277b7..a10382ff41 100644 --- a/specs/core/1_beacon-chain.md +++ b/specs/core/1_beacon-chain.md @@ -142,7 +142,7 @@ class AttestationData(Container): class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] + custody_bits_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -536,7 +536,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe domain=get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits assert len(aggregation_bits) == len(indexed_attestation.committee) - for i, custody_bits in enumerate(attestation.custody_bits): + for i, custody_bits in enumerate(attestation.custody_bits_blocks): assert len(custody_bits) == len(indexed_attestation.committee) for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): if abit: @@ -546,7 +546,10 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit), domain)) else: assert not cbit - + # WARNING: this is BROKEN. If no custody_bits_blocks, + # a valid empty signature can pass validation, even though aggregate bits are set. + # Decide between: force at least 1 shard block (even if empty data), + # or fast-aggregate-verify with attestation data with empty shard data as message (alike to phase0) return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) ``` @@ -616,11 +619,11 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Signature check assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Type 1: on-time attestations - if attestation.custody_bits != []: + if attestation.custody_bits_blocks != []: # Correct slot assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Correct data root count - assert len(attestation.custody_bits) == len(get_offset_slots(state, shard_start_slot)) + assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard_start_slot)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) # Type 2: delayed attestations diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 5de4997ae1..3ca0a29b7a 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -91,7 +91,7 @@ The following types are defined, mapping into `DomainType` (little endian): ```python class CustodySlashing(Container): - # Attestation.custody_bits[data_index][committee.index(malefactor_index)] is the target custody bit to check. + # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. data_index: uint64 malefactor_index: ValidatorIndex @@ -378,7 +378,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed assert bls.Verify(malefactor.pubkey, signing_root, custody_slashing.malefactor_secret) # Get the custody bit - custody_bits = attestation.custody_bits[custody_slashing.data_index] + custody_bits = attestation.custody_bits_blocks[custody_slashing.data_index] committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 20abcacfb8..8a342dd4d8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -16,3 +16,40 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1=spec.get_indexed_attestation(state, attestation_1), attestation_2=spec.get_indexed_attestation(state, attestation_2), ) + + +def get_indexed_attestation_participants(spec, indexed_att): + """ + Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. + """ + if spec.version == "phase1": + return list(spec.get_indices_from_committee( + indexed_att.committee, + indexed_att.attestation.aggregation_bits, + )) + else: + return list(indexed_att.attesting_indices) + + +def set_indexed_attestation_participants(spec, indexed_att, participants): + """ + Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. + """ + if spec.version == "phase1": + indexed_att.attestation.aggregation_bits = [bool(i in participants) for i in indexed_att.committee] + else: + indexed_att.attesting_indices = participants + + +def get_attestation_1_data(spec, att_slashing): + if spec.version == "phase1": + return att_slashing.attestation_1.attestation.data + else: + return att_slashing.attestation_1.data + + +def get_attestation_2_data(spec, att_slashing): + if spec.version == "phase1": + return att_slashing.attestation_2.attestation.data + else: + return att_slashing.attestation_2.data diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 50cd7f706f..4bd3a96b5e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,7 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import sign_indexed_attestation -from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ + get_indexed_attestation_participants, get_attestation_2_data, get_attestation_1_data from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import ( get_balance, @@ -25,7 +26,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', None return - slashed_indices = attester_slashing.attestation_1.attesting_indices + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) @@ -92,12 +93,12 @@ def test_success_surround(spec, state): state.current_justified_checkpoint.epoch += 1 attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attestation_1 = attester_slashing.attestation_1 - attestation_2 = attester_slashing.attestation_2 + att_1_data = get_attestation_1_data(spec, attester_slashing) + att_2_data = get_attestation_2_data(spec, attester_slashing) # set attestion1 to surround attestation 2 - attestation_1.data.source.epoch = attestation_2.data.source.epoch - 1 - attestation_1.data.target.epoch = attestation_2.data.target.epoch + 1 + att_1_data.source.epoch = att_2_data.source.epoch - 1 + att_1_data.target.epoch = att_2_data.target.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -109,7 +110,7 @@ def test_success_surround(spec, state): @always_bls def test_success_already_exited_recent(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - slashed_indices = attester_slashing.attestation_1.attesting_indices + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) for index in slashed_indices: spec.initiate_validator_exit(state, index) @@ -121,7 +122,7 @@ def test_success_already_exited_recent(spec, state): @always_bls def test_success_already_exited_long_ago(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - slashed_indices = attester_slashing.attestation_1.attesting_indices + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) for index in slashed_indices: spec.initiate_validator_exit(state, index) state.validators[index].withdrawable_epoch = spec.get_current_epoch(state) + 2 @@ -158,7 +159,12 @@ def test_invalid_sig_1_and_2(spec, state): def test_same_data(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data = attester_slashing.attestation_2.data + indexed_att_1 = attester_slashing.attestation_1 + att_2_data = get_attestation_2_data(spec, attester_slashing) + if spec.version == 'phase1': + indexed_att_1.attestation.data = att_2_data + else: + indexed_att_1.data = att_2_data sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) @@ -169,10 +175,8 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - if spec.version == 'phase0': - attester_slashing.attestation_1.data.target.epoch += 1 - else: - attester_slashing.attestation_1.attestation.data.target.epoch += 1 + att_1_data = get_attestation_1_data(spec, attester_slashing) + att_1_data.target.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -185,20 +189,23 @@ def test_participants_already_slashed(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) # set all indices to slashed - validator_indices = attester_slashing.attestation_1.attesting_indices + validator_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) for index in validator_indices: state.validators[index].slashed = True yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +# Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list. + + +@with_phases(['phase0']) @spec_state_test @always_bls def test_att1_bad_extra_index(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - indices = attester_slashing.attestation_1.attesting_indices + indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) options = list(set(range(len(state.validators))) - set(indices)) indices.append(options[len(options) // 2]) # add random index, not previously in attestation. attester_slashing.attestation_1.attesting_indices = sorted(indices) @@ -208,7 +215,7 @@ def test_att1_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att1_bad_replaced_index(spec, state): @@ -224,7 +231,7 @@ def test_att1_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att2_bad_extra_index(spec, state): @@ -240,7 +247,7 @@ def test_att2_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att2_bad_replaced_index(spec, state): @@ -256,7 +263,7 @@ def test_att2_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att1_duplicate_index_normal_signed(spec, state): @@ -276,7 +283,7 @@ def test_att1_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att2_duplicate_index_normal_signed(spec, state): @@ -296,7 +303,7 @@ def test_att2_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att1_duplicate_index_double_signed(spec, state): @@ -311,7 +318,7 @@ def test_att1_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test @always_bls def test_att2_duplicate_index_double_signed(spec, state): @@ -326,7 +333,7 @@ def test_att2_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -339,7 +346,7 @@ def test_unsorted_att_1(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0']) @spec_state_test def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index b386d36b40..9027660ab2 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ transition_unsigned_block from eth2spec.test.helpers.keys import privkeys, pubkeys -from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -220,7 +220,7 @@ def test_attester_slashing(spec, state): pre_state = deepcopy(state) attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = attester_slashing.attestation_1.attesting_indices[0] + validator_index = get_indexed_attestation_participants(spec, attester_slashing.attestation_1)[0] assert not state.validators[validator_index].slashed From 507a9afbfb0d434f4c2cd59ef444860f9dce70e8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 Jan 2020 18:57:56 +0100 Subject: [PATCH 093/111] apply custody bit fix suggestion from Dankrad --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index d8c4954f32..fd804a7224 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -188,7 +188,7 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: full_G2_element = bls.signature_to_G2(key) s = full_G2_element[0].coeffs - bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(atom, "little"), BLS12_381_Q) + bits = [legendre_bit(sum(s[i % 2]**i * int.from_bytes(atom, "little")), BLS12_381_Q) for i, atom in enumerate(get_custody_atoms(data))] # XOR all atom bits return bit(sum(bits) % 2) From 419b6a3250b5f33d88b8f4f847701ad13c599b03 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 Jan 2020 19:00:24 +0100 Subject: [PATCH 094/111] config change, need more space for worst-case reveals --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index fd804a7224..1e7dc0216f 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -65,7 +65,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | | - | - | -| `MAX_CUSTODY_KEY_REVEALS` | `2**4` (= 16) | +| `MAX_CUSTODY_KEY_REVEALS` | `2**8` (= 256) | | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | | `MAX_CUSTODY_SLASHINGS` | `1` | From 702b253361ace68d0130d87c4309fc095567e017 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 Jan 2020 19:47:33 +0100 Subject: [PATCH 095/111] update configs for phase1 --- configs/mainnet.yaml | 66 +++++++++++++++++++++++++++++-- configs/minimal.yaml | 75 +++++++++++++++++++++++++++++++----- specs/phase1/beacon-chain.md | 4 +- specs/phase1/custody-game.md | 7 +++- specs/phase1/phase1-fork.md | 2 +- 5 files changed, 135 insertions(+), 19 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 2d8de53905..6eb5641d09 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,9 +94,6 @@ PERSISTENT_COMMITTEE_PERIOD: 2048 MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 -# 2**14 (= 16,384) epochs ~73 days -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 - # State vector lengths @@ -146,12 +143,73 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 +# Phase 1 DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Phase 1 +# Phase 1: Upgrade from Phase 0 # --------------------------------------------------------------- +PHASE_1_FORK_VERSION: 0x01000000 INITIAL_ACTIVE_SHARDS: 64 +# Placeholder +INITIAL_GASPRICE: 10 + + +# Phase 1: General +# --------------------------------------------------------------- +# 2**10` (= 1024) +MAX_SHARDS: 1024 +# 2**3 (= 8) | online epochs | ~51 min +ONLINE_PERIOD: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**8 (= 256) | epochs | ~27 hours +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# 2**8 (= 256) | epochs | ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**18 (= 262,144) +SHARD_BLOCK_CHUNK_SIZE: 262144 +# 2**2 (= 4) +MAX_SHARD_BLOCK_CHUNKS: 4 +# 3 * 2**16` (= 196,608) +TARGET_SHARD_BLOCK_SIZE: 196608 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**5 (= 32) Gwei +MIN_GASPRICE: 32 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + + +# Phase 1: Custody Game +# --------------------------------------------------------------- + +# Time parameters +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**14 (= 16,384) epochs ~73 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 +# 2**11 (= 2,048) epochs, ~9 days +EPOCHS_PER_CUSTODY_PERIOD: 2048 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**7 (= 128) epochs, ~14 hours +MAX_REVEAL_LATENESS_DECREMENT: 128 + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d45bdbd095..4868034671 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -21,14 +21,13 @@ MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 # Jan 3, 2020 MIN_GENESIS_TIME: 1578009600 -# -# + # Fork Choice # --------------------------------------------------------------- # 2**1 (= 1) SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 2 -# + # Validator # --------------------------------------------------------------- # [customized] process deposits more quickly, but insecure @@ -94,12 +93,6 @@ PERSISTENT_COMMITTEE_PERIOD: 2048 MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 -# [customized] 2**12 (= 4,096) epochs -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# 2**2 (= 4) epochs -EPOCHS_PER_CUSTODY_PERIOD: 4 -# 2**2 (= 4) epochs -CUSTODY_PERIOD_TO_RANDAO_PADDING: 4 # State vector lengths @@ -149,13 +142,75 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 +# Phase 1 DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Phase 1 +# Phase 1: Upgrade from Phase 0 # --------------------------------------------------------------- +# [customized] for testnet distinction +PHASE_1_FORK_VERSION: 0x01000001 # [customized] reduced for testing INITIAL_ACTIVE_SHARDS: 4 +# Placeholder +INITIAL_GASPRICE: 10 + + +# Phase 1: General +# --------------------------------------------------------------- +# [customized] reduced for testing +MAX_SHARDS: 8 +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# 2**8 (= 256) | epochs +SHARD_COMMITTEE_PERIOD: 256 +# 2**18 (= 262,144) +SHARD_BLOCK_CHUNK_SIZE: 262144 +# 2**2 (= 4) +MAX_SHARD_BLOCK_CHUNKS: 4 +# 3 * 2**16` (= 196,608) +TARGET_SHARD_BLOCK_SIZE: 196608 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**5 (= 32) Gwei +MIN_GASPRICE: 32 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + + +# Phase 1: Custody Game +# --------------------------------------------------------------- + +# Time parameters +# 2**1 (= 2) epochs +RANDAO_PENALTY_EPOCHS: 2 +# [customized] quicker for testing +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 +# 2**11 (= 2,048) epochs, +EPOCHS_PER_CUSTODY_PERIOD: 2048 +# 2**11 (= 2,048) epochs, +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**7 (= 128) epochs, +MAX_REVEAL_LATENESS_DECREMENT: 128 + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 14b83dd036..b53ce1de1f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -107,7 +107,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `EMPTY_CHUNK_ROOT` | `hash_tree_root(ByteList[SHARD_BLOCK_CHUNK_SIZE]())` | | | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | @@ -406,8 +405,9 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid ```python def chunks_to_body_root(chunks: List[Bytes32, MAX_SHARD_BLOCK_CHUNKS]) -> Root: + empty_chunk_root = hash_tree_root(ByteList[SHARD_BLOCK_CHUNK_SIZE]()) return hash_tree_root(Vector[Bytes32, MAX_SHARD_BLOCK_CHUNKS]( - chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) + chunks + [empty_chunk_root] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks)) )) ``` diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 1e7dc0216f..54a3f43329 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -45,18 +45,20 @@ This document details the beacon chain additions and changes in Phase 1 of Ether ## Constants ### Misc + | Name | Value | Unit | | - | - | | `BLS12_381_Q` | `4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787` | -| `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | | `BYTES_PER_CUSTODY_ATOM` | `48` | bytes | +## Configuration + ### Time parameters | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `RANDAO_PENALTY_EPOCHS` | `2**1` (= 2) | epochs | 12.8 minutes | -| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` | epochs | ~73 days | +| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` (= 16,384) | epochs | ~73 days | | `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | | `MAX_REVEAL_LATENESS_DECREMENT` | `2**7` (= 128) | epochs | ~14 hours | @@ -74,6 +76,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | | - | - | | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | +| `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | ### Signature domain types diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index c6bbc73638..56eee410b0 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -34,7 +34,7 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | - | -| `PHASE_1_FORK_VERSION` | `Version('0x00000001')` | +| `PHASE_1_FORK_VERSION` | `Version('0x01000000')` | | `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | | `INITIAL_GASPRICE` | `Gwei(10)` | From 46be6aed1d26a0227d2f66dfca7b132bafe1e428 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 Jan 2020 20:01:42 +0100 Subject: [PATCH 096/111] fix config loader to support list inputs --- tests/core/config_helpers/preset_loader/loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/core/config_helpers/preset_loader/loader.py b/tests/core/config_helpers/preset_loader/loader.py index 9d75932df6..95f147f6e0 100644 --- a/tests/core/config_helpers/preset_loader/loader.py +++ b/tests/core/config_helpers/preset_loader/loader.py @@ -18,7 +18,9 @@ def load_presets(configs_dir, presets_name) -> Dict[str, Any]: loaded = yaml.load(path) out = dict() for k, v in loaded.items(): - if v.startswith("0x"): + if isinstance(v, list): + out[k] = v + elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) else: out[k] = int(v) From 6c744681422573dec89e3e8909a7b8f5deb71459 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:02:02 +0100 Subject: [PATCH 097/111] forkchoice store on top of any state now --- specs/phase0/fork-choice.md | 34 +++++++++++-------- .../test/fork_choice/test_get_head.py | 10 +++--- .../test/fork_choice/test_on_attestation.py | 20 +++++------ .../test/fork_choice/test_on_block.py | 14 ++++---- .../eth2spec/test/fork_choice/test_on_tick.py | 12 +++---- 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 0d9823fcdb..feab5bb7ae 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -14,7 +14,7 @@ - [Helpers](#helpers) - [`LatestMessage`](#latestmessage) - [`Store`](#store) - - [`get_genesis_store`](#get_genesis_store) + - [`get_forkchoice_store`](#get_forkchoice_store) - [`get_slots_since_genesis`](#get_slots_since_genesis) - [`get_current_slot`](#get_current_slot) - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) @@ -38,7 +38,7 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 ## Fork choice -The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running: +The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_checkpoint_store(genesis_state)` and update `store` by running: - `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time - `on_block(block)` whenever a block `block: SignedBeaconBlock` is received @@ -79,29 +79,35 @@ class Store(object): justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + blocks: Dict[Root, BeaconBlockHeader] = field(default_factory=dict) block_states: Dict[Root, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) ``` -#### `get_genesis_store` +#### `get_forkchoice_store` + +The provided anchor-state will be regarded as a trusted state, to not roll back beyond. +This should be the genesis state for a full client. ```python -def get_genesis_store(genesis_state: BeaconState) -> Store: - genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) - root = hash_tree_root(genesis_block) - justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) - finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) +def get_forkchoice_store(anchor_state: BeaconState) -> Store: + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) return Store( - time=genesis_state.genesis_time, - genesis_time=genesis_state.genesis_time, + time=anchor_state.genesis_time, + genesis_time=anchor_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, best_justified_checkpoint=justified_checkpoint, - blocks={root: genesis_block}, - block_states={root: genesis_state.copy()}, - checkpoint_states={justified_checkpoint: genesis_state.copy()}, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 1f412e7879..a5a6b2fe07 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -34,7 +34,7 @@ def add_attestation_to_store(spec, store, attestation): @spec_state_test def test_genesis(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) assert spec.get_head(store) == spec.hash_tree_root(genesis_block) @@ -43,7 +43,7 @@ def test_genesis(spec, state): @spec_state_test def test_chain_no_attestations(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) assert spec.get_head(store) == spec.hash_tree_root(genesis_block) @@ -66,7 +66,7 @@ def test_split_tie_breaker_no_attestations(spec, state): genesis_state = state.copy() # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) assert spec.get_head(store) == spec.hash_tree_root(genesis_block) @@ -94,7 +94,7 @@ def test_shorter_chain_but_heavier_weight(spec, state): genesis_state = state.copy() # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) assert spec.get_head(store) == spec.hash_tree_root(genesis_block) @@ -123,7 +123,7 @@ def test_shorter_chain_but_heavier_weight(spec, state): def test_filtered_block_tree(spec, state): # Initialization genesis_state_root = state.hash_tree_root() - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) genesis_block = spec.BeaconBlock(state_root=genesis_state_root) # transition state past initial couple of epochs diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index d7fbc4777a..0fa6809ab0 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -27,7 +27,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): @with_all_phases @spec_state_test def test_on_attestation_current_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) spec.on_tick(store, store.time + spec.SECONDS_PER_SLOT * 2) block = build_empty_block_for_next_slot(spec, state) @@ -46,7 +46,7 @@ def test_on_attestation_current_epoch(spec, state): @with_all_phases @spec_state_test def test_on_attestation_previous_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) spec.on_tick(store, store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) block = build_empty_block_for_next_slot(spec, state) @@ -65,7 +65,7 @@ def test_on_attestation_previous_epoch(spec, state): @with_all_phases @spec_state_test def test_on_attestation_past_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) # move time forward 2 epochs time = store.time + 2 * spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH @@ -87,7 +87,7 @@ def test_on_attestation_past_epoch(spec, state): @with_all_phases @spec_state_test def test_on_attestation_mismatched_target_and_slot(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) spec.on_tick(store, store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) block = build_empty_block_for_next_slot(spec, state) @@ -110,7 +110,7 @@ def test_on_attestation_mismatched_target_and_slot(spec, state): @with_all_phases @spec_state_test def test_on_attestation_target_not_in_store(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH spec.on_tick(store, time) @@ -131,7 +131,7 @@ def test_on_attestation_target_not_in_store(spec, state): @with_all_phases @spec_state_test def test_on_attestation_beacon_block_not_in_store(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH spec.on_tick(store, time) @@ -159,7 +159,7 @@ def test_on_attestation_beacon_block_not_in_store(spec, state): @with_all_phases @spec_state_test def test_on_attestation_future_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -179,7 +179,7 @@ def test_on_attestation_future_epoch(spec, state): @with_all_phases @spec_state_test def test_on_attestation_future_block(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = spec.SECONDS_PER_SLOT * 5 spec.on_tick(store, time) @@ -199,7 +199,7 @@ def test_on_attestation_future_block(spec, state): @with_all_phases @spec_state_test def test_on_attestation_same_slot(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 1 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -215,7 +215,7 @@ def test_on_attestation_same_slot(spec, state): @with_all_phases @spec_state_test def test_on_attestation_invalid_attestation(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index 10d1c0011b..6a72d61e1c 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -36,7 +36,7 @@ def apply_next_epoch_with_attestations(spec, state, store): @spec_state_test def test_basic(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) assert store.time == time @@ -60,7 +60,7 @@ def test_basic(spec, state): @spec_state_test def test_on_block_checkpoints(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) @@ -86,7 +86,7 @@ def test_on_block_checkpoints(spec, state): @spec_state_test def test_on_block_future_block(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) # do not tick time @@ -100,7 +100,7 @@ def test_on_block_future_block(spec, state): @spec_state_test def test_on_block_bad_parent_root(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) @@ -120,7 +120,7 @@ def test_on_block_bad_parent_root(spec, state): @spec_state_test def test_on_block_before_finalized(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) @@ -139,7 +139,7 @@ def test_on_block_before_finalized(spec, state): @spec_state_test def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) @@ -170,7 +170,7 @@ def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): @spec_state_test def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state): # Initialization - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) time = 100 spec.on_tick(store, time) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py index 77222f65c6..27b64ac098 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py @@ -19,14 +19,14 @@ def run_on_tick(spec, store, time, new_justified_checkpoint=False): @with_all_phases @spec_state_test def test_basic(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) run_on_tick(spec, store, store.time + 1) @with_all_phases @spec_state_test def test_update_justified_single(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH store.best_justified_checkpoint = spec.Checkpoint( @@ -40,7 +40,7 @@ def test_update_justified_single(spec, state): @with_all_phases @spec_state_test def test_no_update_same_slot_at_epoch_boundary(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH store.best_justified_checkpoint = spec.Checkpoint( @@ -57,7 +57,7 @@ def test_no_update_same_slot_at_epoch_boundary(spec, state): @with_all_phases @spec_state_test def test_no_update_not_epoch_boundary(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) store.best_justified_checkpoint = spec.Checkpoint( epoch=store.justified_checkpoint.epoch + 1, @@ -70,7 +70,7 @@ def test_no_update_not_epoch_boundary(spec, state): @with_all_phases @spec_state_test def test_no_update_new_justified_equal_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH store.best_justified_checkpoint = spec.Checkpoint( @@ -89,7 +89,7 @@ def test_no_update_new_justified_equal_epoch(spec, state): @with_all_phases @spec_state_test def test_no_update_new_justified_later_epoch(spec, state): - store = spec.get_genesis_store(state) + store = spec.get_forkchoice_store(state) seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH store.best_justified_checkpoint = spec.Checkpoint( From fee3baaf8ccc094c78772bae02fb031e7b801eef Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:02:24 +0100 Subject: [PATCH 098/111] fix attesting indices error --- .../epoch_processing/test_process_rewards_and_penalties.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index dd3dae50aa..fa394df564 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -10,6 +10,7 @@ add_attestations_to_state, get_valid_attestation, ) +from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -142,7 +143,7 @@ def test_duplicate_attestation(spec, state): attestation = get_valid_attestation(spec, state, signed=True) indexed_attestation = spec.get_indexed_attestation(state, attestation) - participants = indexed_attestation.attesting_indices + participants = get_indexed_attestation_participants(spec, indexed_attestation) assert len(participants) > 0 From e8654bff1077f3c2580376c73b71f9359e4d1285 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:04:15 +0100 Subject: [PATCH 099/111] merkle proofs out, moving to ssz spec --- .../eth2spec/test/merkle_proofs/__init__.py | 0 .../test/merkle_proofs/test_merkle_proofs.py | 152 ------------------ 2 files changed, 152 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/merkle_proofs/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py diff --git a/tests/core/pyspec/eth2spec/test/merkle_proofs/__init__.py b/tests/core/pyspec/eth2spec/test/merkle_proofs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py b/tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py deleted file mode 100644 index 62a2f63793..0000000000 --- a/tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py +++ /dev/null @@ -1,152 +0,0 @@ -import re -from eth_utils import ( - to_tuple, -) - -from eth2spec.test.context import ( - expect_assertion_error, - spec_state_test, - with_all_phases_except, -) -from eth2spec.utils.ssz.ssz_typing import ( - Bytes32, - Container, - List, - uint64, -) - - -class Foo(Container): - x: uint64 - y: List[Bytes32, 2] - -# Tree -# root -# / \ -# x y_root -# / \ -# y_data_root len(y) -# / \ -# / \ / \ -# -# Generalized indices -# 1 -# / \ -# 2 (x) 3 (y_root) -# / \ -# 6 7 -# / \ -# 12 13 - - -@to_tuple -def ssz_object_to_path(start, end): - is_len = False - len_findall = re.findall(r"(?<=len\().*(?=\))", end) - if len_findall: - is_len = True - end = len_findall[0] - - route = '' - if end.startswith(start): - route = end[len(start):] - - segments = route.split('.') - for word in segments: - index_match = re.match(r"(\w+)\[(\d+)]", word) - if index_match: - yield from index_match.groups() - elif len(word): - yield word - if is_len: - yield '__len__' - - -to_path_test_cases = [ - ('foo', 'foo.x', ('x',)), - ('foo', 'foo.x[100].y', ('x', '100', 'y')), - ('foo', 'foo.x[100].y[1].z[2]', ('x', '100', 'y', '1', 'z', '2')), - ('foo', 'len(foo.x[100].y[1].z[2])', ('x', '100', 'y', '1', 'z', '2', '__len__')), -] - - -def test_to_path(): - for test_case in to_path_test_cases: - start, end, expected = test_case - assert ssz_object_to_path(start, end) == expected - - -generalized_index_cases = [ - (Foo, ('x',), 2), - (Foo, ('y',), 3), - (Foo, ('y', 0), 12), - (Foo, ('y', 1), 13), - (Foo, ('y', '__len__'), None), -] - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_get_generalized_index(spec, state): - for typ, path, generalized_index in generalized_index_cases: - if generalized_index is not None: - assert spec.get_generalized_index( - typ=typ, - path=path, - ) == generalized_index - else: - expect_assertion_error(lambda: spec.get_generalized_index(typ=typ, path=path)) - - yield 'typ', typ - yield 'path', path - yield 'generalized_index', generalized_index - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_verify_merkle_proof(spec, state): - h = spec.hash - a = b'\x11' * 32 - b = b'\x22' * 32 - c = b'\x33' * 32 - d = b'\x44' * 32 - root = h(h(a + b) + h(c + d)) - leaf = a - generalized_index = 4 - proof = [b, h(c + d)] - - is_valid = spec.verify_merkle_proof( - leaf=leaf, - proof=proof, - index=generalized_index, - root=root, - ) - assert is_valid - - yield 'proof', proof - yield 'is_valid', is_valid - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_verify_merkle_multiproof(spec, state): - h = spec.hash - a = b'\x11' * 32 - b = b'\x22' * 32 - c = b'\x33' * 32 - d = b'\x44' * 32 - root = h(h(a + b) + h(c + d)) - leaves = [a, d] - generalized_indices = [4, 7] - proof = [c, b] # helper_indices = [6, 5] - - is_valid = spec.verify_merkle_multiproof( - leaves=leaves, - proof=proof, - indices=generalized_indices, - root=root, - ) - assert is_valid - - yield 'proof', proof - yield 'is_valid', is_valid From f6f8bd535046cd783d40e1a941791e7bae5bedaa Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:34:38 +0100 Subject: [PATCH 100/111] no custody bits fallback --- specs/phase1/beacon-chain.md | 31 +++++++++++-------- .../eth2spec/test/helpers/attestations.py | 1 + .../pyspec/eth2spec/test/helpers/custody.py | 3 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 8 ++--- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index b53ce1de1f..fdd3d5c010 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -536,21 +536,26 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits assert len(aggregation_bits) == len(indexed_attestation.committee) - for i, custody_bits in enumerate(attestation.custody_bits_blocks): - assert len(custody_bits) == len(indexed_attestation.committee) - for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): + + if len(attestation.custody_bits_blocks) == 0: + # fall back on phase0 behavior if there is no shard data. + for participant, abit in zip(indexed_attestation.committee, aggregation_bits): if abit: all_pubkeys.append(state.validators[participant].pubkey) - # Note: only 2N distinct message hashes - all_signing_roots.append(compute_signing_root( - AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit), domain)) - else: - assert not cbit - # WARNING: this is BROKEN. If no custody_bits_blocks, - # a valid empty signature can pass validation, even though aggregate bits are set. - # Decide between: force at least 1 shard block (even if empty data), - # or fast-aggregate-verify with attestation data with empty shard data as message (alike to phase0) - return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) + signing_root = compute_signing_root(indexed_attestation.attestation.data, domain) + return bls.FastAggregateVerify(all_pubkeys, signing_root, signature=attestation.signature) + else: + for i, custody_bits in enumerate(attestation.custody_bits_blocks): + assert len(custody_bits) == len(indexed_attestation.committee) + for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): + if abit: + all_pubkeys.append(state.validators[participant].pubkey) + # Note: only 2N distinct message hashes + all_signing_roots.append(compute_signing_root( + AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit), domain)) + else: + assert not cbit + return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1cd9c07c44..b8733705a6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -77,6 +77,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey ) ) + # TODO: we should try signing custody bits if spec.version == 'phase1' return bls.Aggregate(signatures) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index e00d64a172..bcf2c199b8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,6 +1,5 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls -from eth2spec.utils.hash_function import hash from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector from eth2spec.utils.ssz.ssz_impl import chunkify, pack, hash_tree_root from eth2spec.utils.merkle_minimal import get_merkle_tree, get_merkle_proof @@ -21,7 +20,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) reveal = bls.Sign(privkeys[revealed_index], signing_root) # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) - mask = hash(reveal) + mask = spec.hash(reveal) # Generate masker's signature on the mask signing_root = spec.compute_signing_root(mask, domain) masker_signature = bls.Sign(privkeys[masker_index], signing_root) diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index c7a144ec2f..113bcf169a 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -2,7 +2,7 @@ from ..hash_function import hash from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, ByteList, - Bitlist, Bitvector, uint, + Bitlist, Bitvector, uint, Bytes32 ) # SSZ Serialization @@ -140,7 +140,7 @@ def chunk_count(typ: SSZType) -> int: raise Exception(f"Type not supported: {typ}") -def hash_tree_root(obj: SSZValue): +def hash_tree_root(obj: SSZValue) -> Bytes32: if isinstance(obj, Series): if is_bottom_layer_kind(obj.type()): leaves = chunkify(pack(obj)) @@ -152,6 +152,6 @@ def hash_tree_root(obj: SSZValue): raise Exception(f"Type not supported: {type(obj)}") if isinstance(obj, (List, ByteList, Bitlist)): - return mix_in_length(merkleize_chunks(leaves, limit=chunk_count(obj.type())), len(obj)) + return Bytes32(mix_in_length(merkleize_chunks(leaves, limit=chunk_count(obj.type())), len(obj))) else: - return merkleize_chunks(leaves) + return Bytes32(merkleize_chunks(leaves)) From f04a686db7043cb4d8a1f3c7f079ab9fb0ed0ca9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:42:19 +0100 Subject: [PATCH 101/111] doctoc --- specs/phase1/beacon-chain.md | 4 +++- specs/phase1/custody-game.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fdd3d5c010..c557bf1234 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,8 @@ - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) - [Extended `PendingAttestation`](#extended-pendingattestation) + - [`IndexedAttestation`](#indexedattestation) + - [Extended `AttesterSlashing`](#extended-attesterslashing) - [Extended `Validator`](#extended-validator) - [Extended `BeaconBlockBody`](#extended-beaconblockbody) - [Extended `BeaconBlock`](#extended-beaconblock) @@ -22,7 +24,6 @@ - [`ShardSignableHeader`](#shardsignableheader) - [`ShardState`](#shardstate) - [`ShardTransition`](#shardtransition) - - [`AttestationAndCommittee`](#attestationandcommittee) - [`CompactCommittee`](#compactcommittee) - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - [Helper functions](#helper-functions) @@ -53,6 +54,7 @@ - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - [`process_attestations`](#process_attestations) + - [New Attester slashing processing](#new-attester-slashing-processing) - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 54a3f43329..fd35e6515e 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -11,6 +11,7 @@ - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) +- [Configuration](#configuration) - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) From d98c50a7b1f69f9ecc2809efed2b26467ce771bd Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:43:58 +0100 Subject: [PATCH 102/111] update ci caches --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8189166d3d..23425a1793 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,26 +35,26 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v8-pyspec + venv_name: v9-pyspec reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v8-pyspec + venv_name: v9-pyspec reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} venv_path: ./tests/core/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v9-deposit-contract + venv_name: v10-deposit-contract reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v9-deposit-contract + venv_name: v10-deposit-contract reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 5785b4fc5bcf0337fb073a9b21312379d1f228f1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:59:01 +0100 Subject: [PATCH 103/111] custody bits temporary solution --- specs/phase1/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index c557bf1234..185d27f52e 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -633,9 +633,9 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard_start_slot)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) - # Type 2: delayed attestations + # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. else: - assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH + # assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH assert data.shard_transition_root == Root() ``` From f16d74d703e782280aa413e8c94604d629fc444d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 01:59:27 +0100 Subject: [PATCH 104/111] fix custody key reveal tests not running due to decorator order problem --- .../test_process_custody_key_reveal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py index f8860cf877..fb9157f2f1 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py @@ -55,8 +55,8 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_success(spec, state): state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH custody_key_reveal = get_valid_custody_key_reveal(spec, state) @@ -65,8 +65,8 @@ def test_success(spec, state): @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_reveal_too_early(spec, state): custody_key_reveal = get_valid_custody_key_reveal(spec, state) @@ -74,8 +74,8 @@ def test_reveal_too_early(spec, state): @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_wrong_period(spec, state): custody_key_reveal = get_valid_custody_key_reveal(spec, state, period=5) @@ -83,8 +83,8 @@ def test_wrong_period(spec, state): @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_late_reveal(spec, state): state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH * 3 + 150 custody_key_reveal = get_valid_custody_key_reveal(spec, state) @@ -93,8 +93,8 @@ def test_late_reveal(spec, state): @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_double_reveal(spec, state): state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH * 2 custody_key_reveal = get_valid_custody_key_reveal(spec, state) @@ -105,8 +105,8 @@ def test_double_reveal(spec, state): @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_max_decrement(spec, state): state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH * 3 + 150 custody_key_reveal = get_valid_custody_key_reveal(spec, state) From 0d8a2ef92a21a5b9f2d881b8f698f503f8204c81 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 02:25:59 +0100 Subject: [PATCH 105/111] fix broken fork choice tests; compute anchor root for state, not just any genesis block --- .../test/fork_choice/test_get_head.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index a5a6b2fe07..e34c32c0e9 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -30,13 +30,20 @@ def add_attestation_to_store(spec, store, attestation): spec.on_attestation(store, attestation) +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + @with_all_phases @spec_state_test def test_genesis(spec, state): # Initialization store = spec.get_forkchoice_store(state) - genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) - assert spec.get_head(store) == spec.hash_tree_root(genesis_block) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root @with_all_phases @@ -44,8 +51,8 @@ def test_genesis(spec, state): def test_chain_no_attestations(spec, state): # Initialization store = spec.get_forkchoice_store(state) - genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) - assert spec.get_head(store) == spec.hash_tree_root(genesis_block) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root # On receiving a block of `GENESIS_SLOT + 1` slot block_1 = build_empty_block_for_next_slot(spec, state) @@ -67,8 +74,8 @@ def test_split_tie_breaker_no_attestations(spec, state): # Initialization store = spec.get_forkchoice_store(state) - genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) - assert spec.get_head(store) == spec.hash_tree_root(genesis_block) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root # block at slot 1 block_1_state = genesis_state.copy() @@ -95,8 +102,8 @@ def test_shorter_chain_but_heavier_weight(spec, state): # Initialization store = spec.get_forkchoice_store(state) - genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) - assert spec.get_head(store) == spec.hash_tree_root(genesis_block) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root # build longer tree long_state = genesis_state.copy() @@ -122,15 +129,14 @@ def test_shorter_chain_but_heavier_weight(spec, state): @spec_state_test def test_filtered_block_tree(spec, state): # Initialization - genesis_state_root = state.hash_tree_root() store = spec.get_forkchoice_store(state) - genesis_block = spec.BeaconBlock(state_root=genesis_state_root) + anchor_root = get_anchor_root(spec, state) # transition state past initial couple of epochs next_epoch(spec, state) next_epoch(spec, state) - assert spec.get_head(store) == spec.hash_tree_root(genesis_block) + assert spec.get_head(store) == anchor_root # fill in attestations for entire epoch, justifying the recent epoch prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False) From e6f98eb041693ab68d635bd645352ac339125ab5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 02:26:15 +0100 Subject: [PATCH 106/111] minor config style fixes --- configs/minimal.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 4868034671..a8ea385e73 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -197,11 +197,11 @@ GASPRICE_ADJUSTMENT_COEFFICIENT: 8 RANDAO_PENALTY_EPOCHS: 2 # [customized] quicker for testing EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# 2**11 (= 2,048) epochs, +# 2**11 (= 2,048) epochs EPOCHS_PER_CUSTODY_PERIOD: 2048 -# 2**11 (= 2,048) epochs, +# 2**11 (= 2,048) epochs CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs, +# 2**7 (= 128) epochs MAX_REVEAL_LATENESS_DECREMENT: 128 # Max operations From 90bc1f558b7f4d7654add5917ab24d907edf12e7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 02:26:29 +0100 Subject: [PATCH 107/111] Fix config loading problem --- deposit_contract/requirements-testing.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index cbf6983c11..b8ebe3b848 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -2,4 +2,5 @@ eth-tester[py-evm]==0.1.0b39 git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13 web3==5.0.0b2 pytest==3.6.1 +../tests/core/config_helpers ../tests/core/pyspec From 8d0e1bda6e339322c89fbd57bd449e4a4644cb30 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 14 Jan 2020 02:32:26 +0100 Subject: [PATCH 108/111] downgrade deposit contract back to python 3.6, vyper syntax parsing is broken on v3.8 --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23425a1793..83a82f62ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v10-deposit-contract + venv_name: v11-deposit-contract reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v10-deposit-contract + venv_name: v11-deposit-contract reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: @@ -134,7 +134,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -146,7 +146,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: From 3c07b2c954909d2c5f4d3b8ea35a22bd659103a4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 15 Jan 2020 16:03:07 -0700 Subject: [PATCH 109/111] Make phase 0 fork choice more modular to more easily adopt for slight modifications in phase 1 --- Makefile | 2 +- scripts/build_spec.py | 11 ++- specs/phase0/fork-choice.md | 97 ++++++++++++------- specs/phase1/fork-choice.md | 52 ++++++++++ .../test/fork_choice/test_on_attestation.py | 11 ++- 5 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 specs/phase1/fork-choice.md diff --git a/Makefile b/Makefile index b2ea88e2f7..93f1a9bda2 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE0_SPEC_DIR)/validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/beacon-chain.md $(PHASE1_SPEC_DIR)/fraud-proofs.md $(PHASE1_SPEC_DIR)/phase1-fork.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/beacon-chain.md $(PHASE1_SPEC_DIR)/fraud-proofs.md $(PHASE1_SPEC_DIR)/fork-choice.md $(PHASE1_SPEC_DIR)/phase1-fork.md $@ # TODO: also build validator spec and light-client-sync diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 1831cfa34d..90e9b3fb43 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -230,6 +230,7 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, phase1_custody_sourcefile: str, phase1_beacon_sourcefile: str, phase1_fraud_sourcefile: str, + phase1_fork_choice_sourcefile: str, phase1_fork_sourcefile: str, outfile: str=None) -> Optional[str]: all_sourcefiles = ( @@ -238,6 +239,7 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, phase1_custody_sourcefile, phase1_beacon_sourcefile, phase1_fraud_sourcefile, + phase1_fork_choice_sourcefile, phase1_fork_sourcefile, ) all_spescs = [get_spec(spec) for spec in all_sourcefiles] @@ -267,8 +269,9 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, 3rd argument is input phase1/custody-game.md 4th argument is input phase1/beacon-chain.md 5th argument is input phase1/fraud-proofs.md - 6th argument is input phase1/phase1-fork.md - 7th argument is output spec.py + 6th argument is input phase1/fork-choice.md + 7th argument is input phase1/phase1-fork.md + 8th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -281,13 +284,13 @@ def build_phase1_spec(phase0_beacon_sourcefile: str, else: print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: - if len(args.files) == 7: + if len(args.files) == 8: build_phase1_spec(*args.files) else: print( " Phase 1 requires input files as well as an output file:\n" "\t phase0: (beacon-chain.md, fork-choice.md)\n" - "\t phase1: (custody-game.md, beacon-chain.md, fraud-proofs.md, phase1-fork.md)\n" + "\t phase1: (custody-game.md, beacon-chain.md, fraud-proofs.md, fork-choice.md, phase1-fork.md)\n" "\t and output.py" ) else: diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index feab5bb7ae..e2f24705ea 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -24,6 +24,10 @@ - [`get_filtered_block_tree`](#get_filtered_block_tree) - [`get_head`](#get_head) - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) + - [`on_attestation` helpers](#on_attestation-helpers) + - [`validate_on_attestation`](#validate_on_attestation) + - [`store_target_checkpoint_state`](#store_target_checkpoint_state) + - [`update_latest_messages`](#update_latest_messages) - [Handlers](#handlers) - [`on_tick`](#on_tick) - [`on_block`](#on_block) @@ -257,6 +261,59 @@ def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: C return True ``` +#### `on_attestation` helpers + +##### `validate_on_attestation` + +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives + assert get_current_slot(store) >= compute_start_slot_at_epoch(target.epoch) + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` + +##### `store_target_checkpoint_state` + +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = store.block_states[target.root].copy() + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` + +##### `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` + + ### Handlers #### `on_tick` @@ -323,42 +380,14 @@ def on_attestation(store: Store, attestation: Attestation) -> None: An ``attestation`` that is asserted as invalid may be valid at a later time, consider scheduling it for later processing in such case. """ - target = attestation.data.target - - # Attestations must be from the current or previous epoch - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - # Use GENESIS_EPOCH for previous when genesis to avoid underflow - previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH - assert target.epoch in [current_epoch, previous_epoch] - assert target.epoch == compute_epoch_at_slot(attestation.data.slot) - - # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found - assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives - base_state = store.block_states[target.root].copy() - assert get_current_slot(store) >= compute_start_slot_at_epoch(target.epoch) + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) - # Attestations must be for a known block. If block is unknown, delay consideration until the block is found - assert attestation.data.beacon_block_root in store.blocks - # Attestations must not be for blocks in the future. If not, the attestation should not be considered - assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - - # Store target checkpoint state if not yet seen - if target not in store.checkpoint_states: - process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) - store.checkpoint_states[target] = base_state - target_state = store.checkpoint_states[target] - - # Attestations can only affect the fork choice of subsequent slots. - # Delay consideration in the fork choice until their slot is in the past. - assert get_current_slot(store) >= attestation.data.slot + 1 - - # Get state at the `target` to validate attestation and calculate the committees + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] indexed_attestation = get_indexed_attestation(target_state, attestation) assert is_valid_indexed_attestation(target_state, indexed_attestation) - # Update latest messages - for i in indexed_attestation.attesting_indices: - if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: - store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root) + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) ``` diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md new file mode 100644 index 0000000000..d8bf7fa090 --- /dev/null +++ b/specs/phase1/fork-choice.md @@ -0,0 +1,52 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Handlers](#handlers) + + + + +## Introduction + +This document is the beacon chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_attestation` must be re-specified to handle this. The bulk of `on_attestation` has been moved out into a few helpers to reduce code duplication where possible. + +The rest of the fork choice remains stable. + +### Handlers + +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + attesting_indices = [ + index for i, index in enumerate(indexed_attestation.committee) + if attestation.aggregation_bits[i] + ] + update_latest_messages(store, attesting_indices, attestation) +``` \ No newline at end of file diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 0fa6809ab0..a0a33ca50c 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -15,8 +15,17 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) + + if spec.version == 'phase0': + sample_index = indexed_attestation.attesting_indices[0] + else: + attesting_indices = [ + index for i, index in enumerate(indexed_attestation.committee) + if attestation.aggregation_bits[i] + ] + sample_index = attesting_indices[0] assert ( - store.latest_messages[indexed_attestation.attesting_indices[0]] == + store.latest_messages[sample_index] == spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, From c0b69e531f3d10aa5a90a5a6461b8178e7b62c0f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 15 Jan 2020 17:43:11 -0700 Subject: [PATCH 110/111] cycle through committee indexes instead of through active shards when forming crosslinks --- specs/phase1/beacon-chain.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 185d27f52e..06b66eca57 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -413,6 +413,14 @@ def chunks_to_body_root(chunks: List[Bytes32, MAX_SHARD_BLOCK_CHUNKS]) -> Root: )) ``` +#### `compute_shard_from_committee_index` + +```python +def compute_shard_from_committee_index(state: BeaconState, index: CommitteeIndex, slot: Slot) -> Shard: + active_shards = get_active_shard_count(state) + return Shard((index + get_start_shard(state, slot)) % active_shards) +``` + ### Beacon state accessors #### `get_active_shard_count` @@ -501,8 +509,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: - active_shards = get_active_shard_count(state) - return Shard((attestation.data.index + get_start_shard(state, attestation.data.slot)) % active_shards) + return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` #### `get_next_slot_for_shard` @@ -760,7 +767,9 @@ def process_crosslinks(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Root]]: winners: Set[Tuple[Shard, Root]] = set() - for shard in map(Shard, range(get_active_shard_count(state))): + committee_count = get_committee_count_at_slot(state, state.slot) + for committee_index in map(CommitteeIndex, range(committee_count)): + shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this shard shard_attestations = [ attestation for attestation in attestations From 7a412534d94a8c0c66a0630450192f32fb4d1ca8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 15 Jan 2020 18:16:09 -0700 Subject: [PATCH 111/111] remove test_shard_blocks (outdated) and reduce PERSISTENT_COMMITTEE_PERIOD in minimal config --- configs/minimal.yaml | 4 +- specs/phase1/beacon-chain.md | 1 + .../test/phase_1/sanity/test_shard_blocks.py | 177 ------------------ 3 files changed, 3 insertions(+), 179 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py diff --git a/configs/minimal.yaml b/configs/minimal.yaml index a8ea385e73..03ffa90e36 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -87,8 +87,8 @@ SLOTS_PER_ETH1_VOTING_PERIOD: 16 SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**11 (= 2,048) epochs -PERSISTENT_COMMITTEE_PERIOD: 2048 +# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit +PERSISTENT_COMMITTEE_PERIOD: 128 # [customized] fast catchup crosslinks MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 06b66eca57..2944596733 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -32,6 +32,7 @@ - [`pack_compact_validator`](#pack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`chunks_to_body_root`](#chunks_to_body_root) + - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [Beacon state accessors](#beacon-state-accessors) - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py deleted file mode 100644 index 43b0c56c24..0000000000 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py +++ /dev/null @@ -1,177 +0,0 @@ -from copy import deepcopy - -from eth2spec.test.helpers.phase1.shard_block import ( - build_empty_shard_block, - sign_shard_block, -) -from eth2spec.test.helpers.phase1.shard_state import ( - configure_shard_state, - shard_state_transition_and_sign_block, -) -from eth2spec.test.context import ( - always_bls, - expect_assertion_error, - spec_state_test, - with_all_phases_except, -) - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_process_empty_shard_block(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - block = build_empty_shard_block( - spec, - beacon_state, - shard_state, - slot=shard_state.slot + 1, - signed=True, - full_attestation=False, - ) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - - shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) - - yield 'blocks', [block] - yield 'post', shard_state - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_process_full_attestation_shard_block(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - block = build_empty_shard_block( - spec, - beacon_state, - shard_state, - slot=shard_state.slot + 1, - signed=True, - full_attestation=True, - ) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - - shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) - - yield 'blocks', [block] - yield 'post', shard_state - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_prev_slot_block_transition(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - # Go to clean slot - spec.process_shard_slots(shard_state, shard_state.slot + 1) - # Make a block for it - block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot, signed=True) - # Transition to next slot, above block will not be invalid on top of new state. - spec.process_shard_slots(shard_state, shard_state.slot + 1) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - expect_assertion_error( - lambda: spec.shard_state_transition(beacon_state, shard_state, block) - ) - yield 'blocks', [block] - yield 'post', None - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_same_slot_block_transition(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - # Same slot on top of pre-state, but move out of slot 0 first. - spec.process_shard_slots(shard_state, shard_state.slot + 1) - block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot, signed=True) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - - shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) - - yield 'blocks', [block] - yield 'post', shard_state - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_invalid_state_root(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - spec.process_shard_slots(shard_state, shard_state.slot + 1) - block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot) - block.state_root = b'\x36' * 32 - sign_shard_block(spec, beacon_state, shard_state, block) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - expect_assertion_error( - lambda: spec.shard_state_transition(beacon_state, shard_state, block, validate_state_root=True) - ) - yield 'blocks', [block] - yield 'post', None - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_skipped_slots(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot + 3, signed=True) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - - shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) - - yield 'blocks', [block] - yield 'post', shard_state - - assert shard_state.slot == block.slot - latest_block_header = deepcopy(shard_state.latest_block_header) - latest_block_header.state_root = shard_state.hash_tree_root() - assert latest_block_header.hash_tree_root() == block.hash_tree_root() - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_empty_shard_period_transition(spec, state): - beacon_state, shard_state = configure_shard_state(spec, state) - - # modify some of the deltas to ensure the period transition works properly - stub_delta = 10 - shard_state.newer_committee_positive_deltas[0] = stub_delta - shard_state.newer_committee_negative_deltas[0] = stub_delta - - slot = shard_state.slot + spec.SHARD_SLOTS_PER_EPOCH * spec.EPOCHS_PER_SHARD_PERIOD - beacon_state.slot = spec.compute_epoch_of_shard_slot(slot) * spec.SLOTS_PER_EPOCH - 4 - spec.process_slots(beacon_state, spec.compute_epoch_of_shard_slot(slot) * spec.SLOTS_PER_EPOCH) - - # all validators get slashed for not revealing keys - # undo this to allow for a block proposal - for index in range(len(beacon_state.validators)): - beacon_state.validators[index].slashed = False - block = build_empty_shard_block(spec, beacon_state, shard_state, slot=slot, signed=True) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - - shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) - - yield 'blocks', [block] - yield 'post', shard_state - - shard_state.older_committee_positive_deltas[0] == stub_delta - shard_state.older_committee_negative_deltas[0] == stub_delta - shard_state.newer_committee_positive_deltas[0] == 0 - shard_state.newer_committee_negative_deltas[0] == 0