diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index 23889fd18e..0f56b8bdfa 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 # `uint64(6)` MAX_BLOBS_PER_BLOCK: 6 +# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 +KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 diff --git a/presets/minimal/deneb.yaml b/presets/minimal/deneb.yaml index 9721bec14a..bc4fe4369a 100644 --- a/presets/minimal/deneb.yaml +++ b/presets/minimal/deneb.yaml @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 # `uint64(6)` MAX_BLOBS_PER_BLOCK: 6 +# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 +KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 diff --git a/pysetup/helpers.py b/pysetup/helpers.py index 692aaa0d7e..b2fe00f75e 100644 --- a/pysetup/helpers.py +++ b/pysetup/helpers.py @@ -68,8 +68,7 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str if k in [ "ceillog2", "floorlog2", - "compute_merkle_proof_for_block_body", - "compute_merkle_proof_for_state", + "compute_merkle_proof", ]: del spec_object.functions[k] @@ -111,8 +110,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str: return out # Merge all constant objects - hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {}) + hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {}) hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {}) + hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {}) # Concatenate all strings imports = reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name) ).strip("\n"), builders, "") preparations = reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations() ).strip("\n"), builders, "") @@ -126,6 +126,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str: ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants)) ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants)) custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants)) + func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets)) spec_strs = [ imports, preparations, @@ -147,6 +148,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str: # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. ssz_dep_constants_verification, + func_dep_presets_verification, ] return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n" @@ -223,6 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars) config_vars = combine_dicts(spec0.config_vars, spec1.config_vars) ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants) + func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets) ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types) dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses) return SpecObject( @@ -233,6 +236,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: preset_vars=preset_vars, config_vars=config_vars, ssz_dep_constants=ssz_dep_constants, + func_dep_presets=func_dep_presets, ssz_objects=ssz_objects, dataclasses=dataclasses, ) diff --git a/pysetup/spec_builders/altair.py b/pysetup/spec_builders/altair.py index 4b35380de9..abd9b56c40 100644 --- a/pysetup/spec_builders/altair.py +++ b/pysetup/spec_builders/altair.py @@ -27,16 +27,16 @@ def preparations(cls): @classmethod def sundry_functions(cls) -> str: return ''' -def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex: +def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex: ssz_path = Path(ssz_class) for item in path: ssz_path = ssz_path / item return GeneralizedIndex(ssz_path.gindex()) -def compute_merkle_proof_for_state(state: BeaconState, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(state.get_backing(), index)''' +def compute_merkle_proof(object: SSZObject, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(object.get_backing(), index)''' @classmethod diff --git a/pysetup/spec_builders/base.py b/pysetup/spec_builders/base.py index 44743682ab..ad9a2cb4c4 100644 --- a/pysetup/spec_builders/base.py +++ b/pysetup/spec_builders/base.py @@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # """ return {} + @classmethod + def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: + return {} + @classmethod def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: return functions diff --git a/pysetup/spec_builders/capella.py b/pysetup/spec_builders/capella.py index 03b619b66a..080bdb537f 100644 --- a/pysetup/spec_builders/capella.py +++ b/pysetup/spec_builders/capella.py @@ -13,15 +13,6 @@ def imports(cls, preset_name: str): from eth2spec.bellatrix import {preset_name} as bellatrix ''' - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def compute_merkle_proof_for_block_body(body: BeaconBlockBody, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(body.get_backing(), index)''' - - @classmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: return { diff --git a/pysetup/spec_builders/deneb.py b/pysetup/spec_builders/deneb.py index c32bee8305..e1ad051849 100644 --- a/pysetup/spec_builders/deneb.py +++ b/pysetup/spec_builders/deneb.py @@ -1,3 +1,4 @@ +from typing import Dict from .base import BaseSpecBuilder from ..constants import DENEB @@ -23,7 +24,8 @@ def sundry_functions(cls) -> str: return ''' def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]: # pylint: disable=unused-argument - return [], []''' + return [], [] +''' @classmethod def execution_engine_cls(cls) -> str: @@ -63,9 +65,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine, @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: return { 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, + 'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value, + } + + @classmethod + def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: + return { + 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value, } diff --git a/pysetup/typing.py b/pysetup/typing.py index 56f0cf3b10..28c2220c50 100644 --- a/pysetup/typing.py +++ b/pysetup/typing.py @@ -22,6 +22,7 @@ class SpecObject(NamedTuple): preset_vars: Dict[str, VariableDefinition] config_vars: Dict[str, VariableDefinition] ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects + func_dep_presets: Dict[str, str] # the constants that depend on functions ssz_objects: Dict[str, str] dataclasses: Dict[str, str] diff --git a/setup.py b/setup.py index f540094348..87a3dc3ca8 100644 --- a/setup.py +++ b/setup.py @@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr preset_vars: Dict[str, VariableDefinition] = {} config_vars: Dict[str, VariableDefinition] = {} ssz_dep_constants: Dict[str, str] = {} + func_dep_presets: Dict[str, str] = {} ssz_objects: Dict[str, str] = {} dataclasses: Dict[str, str] = {} custom_types: Dict[str, str] = {} @@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr value_cell = cells[1] value = value_cell.children[0].children + + description = None + if len(cells) >= 3: + description_cell = cells[2] + if len(description_cell.children) > 0: + description = description_cell.children[0].children + if isinstance(description, list): + # marko parses `**X**` as a list containing a X + description = description[0].children + if isinstance(value, list): # marko parses `**X**` as a list containing a X value = value[0].children @@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr ssz_dep_constants[name] = value continue + if description is not None and description.startswith(""): + func_dep_presets[name] = value + value_def = _parse_value(name, value) if name in preset: preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None) @@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr preset_vars=preset_vars, config_vars=config_vars, ssz_dep_constants=ssz_dep_constants, + func_dep_presets=func_dep_presets, ssz_objects=ssz_objects, dataclasses=dataclasses, ) diff --git a/specs/altair/light-client/full-node.md b/specs/altair/light-client/full-node.md index 7dc25448c1..27651af01f 100644 --- a/specs/altair/light-client/full-node.md +++ b/specs/altair/light-client/full-node.md @@ -10,7 +10,7 @@ - [Introduction](#introduction) - [Helper functions](#helper-functions) - - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state) + - [`compute_merkle_proof`](#compute_merkle_proof) - [`block_to_light_client_header`](#block_to_light_client_header) - [Deriving light client data](#deriving-light-client-data) - [`create_light_client_bootstrap`](#create_light_client_bootstrap) @@ -27,11 +27,13 @@ This document provides helper functions to enable full nodes to serve light clie ## Helper functions -### `compute_merkle_proof_for_state` +### `compute_merkle_proof` + +This function return the Merkle proof of the given SSZ object `object` at generalized index `index`. ```python -def compute_merkle_proof_for_state(state: BeaconState, - index: GeneralizedIndex) -> Sequence[Bytes32]: +def compute_merkle_proof(object: SSZObject, + index: GeneralizedIndex) -> Sequence[Bytes32]: ... ``` @@ -73,7 +75,7 @@ def create_light_client_bootstrap(state: BeaconState, return LightClientBootstrap( header=block_to_light_client_header(block), current_sync_committee=state.current_sync_committee, - current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX), + current_sync_committee_branch=compute_merkle_proof(state, CURRENT_SYNC_COMMITTEE_INDEX), ) ``` @@ -120,8 +122,7 @@ def create_light_client_update(state: BeaconState, # `next_sync_committee` is only useful if the message is signed by the current sync committee if update_attested_period == update_signature_period: update.next_sync_committee = attested_state.next_sync_committee - update.next_sync_committee_branch = compute_merkle_proof_for_state( - attested_state, NEXT_SYNC_COMMITTEE_INDEX) + update.next_sync_committee_branch = compute_merkle_proof(attested_state, NEXT_SYNC_COMMITTEE_INDEX) # Indicate finality whenever possible if finalized_block is not None: @@ -130,8 +131,7 @@ def create_light_client_update(state: BeaconState, assert hash_tree_root(update.finalized_header.beacon) == attested_state.finalized_checkpoint.root else: assert attested_state.finalized_checkpoint.root == Bytes32() - update.finality_branch = compute_merkle_proof_for_state( - attested_state, FINALIZED_ROOT_INDEX) + update.finality_branch = compute_merkle_proof(attested_state, FINALIZED_ROOT_INDEX) update.sync_aggregate = block.message.body.sync_aggregate update.signature_slot = block.message.slot diff --git a/specs/capella/light-client/full-node.md b/specs/capella/light-client/full-node.md index a406cd771b..c59af8ec74 100644 --- a/specs/capella/light-client/full-node.md +++ b/specs/capella/light-client/full-node.md @@ -10,7 +10,6 @@ - [Introduction](#introduction) - [Helper functions](#helper-functions) - - [`compute_merkle_proof_for_block_body`](#compute_merkle_proof_for_block_body) - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) @@ -22,14 +21,6 @@ This upgrade adds information about the execution payload to light client data a ## Helper functions -### `compute_merkle_proof_for_block_body` - -```python -def compute_merkle_proof_for_block_body(body: BeaconBlockBody, - index: GeneralizedIndex) -> Sequence[Bytes32]: - ... -``` - ### Modified `block_to_light_client_header` ```python @@ -55,7 +46,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), ) - execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) + execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX) else: # Note that during fork transitions, `finalized_header` may still point to earlier forks. # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 281348167a..18b97ae43a 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -52,7 +52,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: execution_header.blob_gas_used = payload.blob_gas_used execution_header.excess_blob_gas = payload.excess_blob_gas - execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) + execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX) else: # Note that during fork transitions, `finalized_header` may still point to earlier forks. # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index c40c5e0653..17e673c50e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -11,13 +11,14 @@ The specification of these changes continues in the same format as the network s - [Modifications in Deneb](#modifications-in-deneb) + - [Constant](#constant) + - [Preset](#preset) - [Configuration](#configuration) - [Containers](#containers) - [`BlobSidecar`](#blobsidecar) - - [`SignedBlobSidecar`](#signedblobsidecar) - [`BlobIdentifier`](#blobidentifier) - [Helpers](#helpers) - - [`verify_blob_sidecar_signature`](#verify_blob_sidecar_signature) + - [`verify_blob_sidecar_inclusion_proof`](#verify_blob_sidecar_inclusion_proof) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) @@ -41,6 +42,18 @@ The specification of these changes continues in the same format as the network s ## Modifications in Deneb +### Constant + +*[New in Deneb:EIP4844]* + +### Preset + +*[New in Deneb:EIP4844]* + +| Name | Value | Description | +|------------------------------------------|-----------------------------------|---------------------------------------------------------------------| +| `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` | `uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK))` (= 17) | Merkle proof depth for `blob_kzg_commitments` list item | + ### Configuration *[New in Deneb:EIP4844]* @@ -60,24 +73,12 @@ The specification of these changes continues in the same format as the network s ```python class BlobSidecar(Container): - block_root: Root index: BlobIndex # Index of blob in block - slot: Slot - block_parent_root: Root # Proposer shuffling determinant - proposer_index: ValidatorIndex blob: Blob kzg_commitment: KZGCommitment kzg_proof: KZGProof # Allows for quick verification of kzg_commitment -``` - -#### `SignedBlobSidecar` - -*[New in Deneb:EIP4844]* - -```python -class SignedBlobSidecar(Container): - message: BlobSidecar - signature: BLSSignature + signed_block_header: SignedBeaconBlockHeader + kzg_commitment_inclusion_proof: Vector[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH] ``` #### `BlobIdentifier` @@ -92,13 +93,18 @@ class BlobIdentifier(Container): #### Helpers -##### `verify_blob_sidecar_signature` +##### `verify_blob_sidecar_inclusion_proof` ```python -def verify_blob_sidecar_signature(state: BeaconState, signed_blob_sidecar: SignedBlobSidecar) -> bool: - proposer = state.validators[signed_blob_sidecar.message.proposer_index] - signing_root = compute_signing_root(signed_blob_sidecar.message, get_domain(state, DOMAIN_BLOB_SIDECAR)) - return bls.Verify(proposer.pubkey, signing_root, signed_blob_sidecar.signature) +def verify_blob_sidecar_inclusion_proof(blob_sidecar: BlobSidecar) -> bool: + gindex = get_subtree_index(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', blob_sidecar.index)) + return is_valid_merkle_branch( + leaf=blob_sidecar.kzg_commitment.hash_tree_root(), + branch=blob_sidecar.kzg_commitment_inclusion_proof, + depth=KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + index=gindex, + root=blob_sidecar.signed_block_header.message.body_root, + ) ``` ### The gossip domain: gossipsub @@ -123,7 +129,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar | Name | Message Type | | - | - | -| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` [New in Deneb:EIP4844] | +| `blob_sidecar_{subnet_id}` | `BlobSidecar` [New in Deneb:EIP4844] | ##### Global topics @@ -146,19 +152,21 @@ New validation: This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. -The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: - -- _[REJECT]_ The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` -- i.e. `sidecar.index < MAX_BLOBS_PER_BLOCK`. -- _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(sidecar.index) == subnet_id`. -- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot). -- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` -- _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved). -- _[REJECT]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) passes validation. -- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `sidecar.block_parent_root`). -- _[REJECT]_ The current finalized_checkpoint is an ancestor of the sidecar's block's parent -- i.e. `get_checkpoint_block(store, sidecar.block_parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`. -- _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid as verified by `verify_blob_sidecar_signature`. -- _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple `(sidecar.block_root, sidecar.index)`. -- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_parent_root`/`slot`). +The following validations MUST pass before forwarding the `blob_sidecar` on the network, assuming the alias `block_header = blob_sidecar.signed_block_header.message`: + +- _[REJECT]_ The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` -- i.e. `blob_sidecar.index < MAX_BLOBS_PER_BLOCK`. +- _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id`. +- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `block_header.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot). +- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `block_header.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` +- _[REJECT]_ The proposer signature of `blob_sidecar.signed_block_header`, is valid with respect to the `block_header.proposer_index` pubkey. +- _[IGNORE]_ The sidecar's block's parent (defined by `block_header.parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved). +- _[REJECT]_ The sidecar's block's parent (defined by `block_header.parent_root`) passes validation. +- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `block_header.parent_root`). +- _[REJECT]_ The current finalized_checkpoint is an ancestor of the sidecar's block -- i.e. `get_checkpoint_block(store, block_header.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`. +- _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. +- _[REJECT]_ The sidecar's blob is valid as verified by `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. +- _[IGNORE]_ The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, blob_sidecar.index) with valid header signature, sidecar inclusion proof, and kzg proof. +- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. ###### `beacon_aggregate_and_proof` diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 2a8c832c73..3e2c91f817 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -21,7 +21,7 @@ - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [ExecutionPayload](#executionpayload) - [Blob KZG commitments](#blob-kzg-commitments) - - [Constructing the `SignedBlobSidecar`s](#constructing-the-signedblobsidecars) + - [Constructing the `BlobSidecar`s](#constructing-the-blobsidecars) - [Sidecar](#sidecar) @@ -131,51 +131,48 @@ def prepare_execution_payload(state: BeaconState, 1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `blobs_bundle` entry containing the corresponding `blobs`, `commitments`, and `proofs`. 2. Set `block.body.blob_kzg_commitments = commitments`. -#### Constructing the `SignedBlobSidecar`s +#### Constructing the `BlobSidecar`s *[New in Deneb:EIP4844]* -To construct a `SignedBlobSidecar`, a `signed_blob_sidecar` is defined with the necessary context for block and sidecar proposal. +To construct a `BlobSidecar`, a `blob_sidecar` is defined with the necessary context for block and sidecar proposal. ##### Sidecar -Blobs associated with a block are packaged into sidecar objects for distribution to the network. +Blobs associated with a block are packaged into sidecar objects for distribution to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic. Each `sidecar` is obtained from: ```python -def get_blob_sidecars(block: BeaconBlock, +def get_blob_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob], blob_kzg_proofs: Sequence[KZGProof]) -> Sequence[BlobSidecar]: + block = signed_block.message + block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=block.state_root, + body_root=hash_tree_root(block.body), + ) + signed_block_header = SignedBeaconBlockHeader(message=block_header, signature=signed_block.signature) return [ BlobSidecar( - block_root=hash_tree_root(block), index=index, - slot=block.slot, - block_parent_root=block.parent_root, blob=blob, kzg_commitment=block.body.blob_kzg_commitments[index], kzg_proof=blob_kzg_proofs[index], + signed_block_header=signed_block_header, + kzg_commitment_inclusion_proof=compute_merkle_proof( + block.body, + get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', index), + ), ) for index, blob in enumerate(blobs) ] - -``` - -Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic. - -`signature` is obtained from: - -```python -def get_blob_sidecar_signature(state: BeaconState, - sidecar: BlobSidecar, - privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BLOB_SIDECAR, compute_epoch_at_slot(sidecar.slot)) - signing_root = compute_signing_root(sidecar, domain) - return bls.Sign(privkey, signing_root) ``` -The `subnet_id` for the `signed_sidecar` is calculated with: -- Let `blob_index = signed_sidecar.message.index`. +The `subnet_id` for the `blob_sidecar` is calculated with: +- Let `blob_index = blob_sidecar.index`. - Let `subnet_id = compute_subnet_for_blob_sidecar(blob_index)`. ```python diff --git a/ssz/merkle-proofs.md b/ssz/merkle-proofs.md index 6772026fe1..919ff07f59 100644 --- a/ssz/merkle-proofs.md +++ b/ssz/merkle-proofs.md @@ -167,7 +167,7 @@ def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariab ``` ```python -def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: +def get_generalized_index(typ: SSZType, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex: """ Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py index 7e9c6b7e2f..104d4774cf 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py @@ -10,8 +10,7 @@ @spec_state_test def test_current_sync_committee_merkle_proof(spec, state): yield "object", state - current_sync_committee_branch = spec.compute_merkle_proof_for_state( - state, spec.CURRENT_SYNC_COMMITTEE_INDEX) + current_sync_committee_branch = spec.compute_merkle_proof(state, spec.CURRENT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.current_sync_committee.hash_tree_root().hex(), "leaf_index": spec.CURRENT_SYNC_COMMITTEE_INDEX, @@ -31,8 +30,7 @@ def test_current_sync_committee_merkle_proof(spec, state): @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): yield "object", state - next_sync_committee_branch = spec.compute_merkle_proof_for_state( - state, spec.NEXT_SYNC_COMMITTEE_INDEX) + next_sync_committee_branch = spec.compute_merkle_proof(state, spec.NEXT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.next_sync_committee.hash_tree_root().hex(), "leaf_index": spec.NEXT_SYNC_COMMITTEE_INDEX, @@ -52,8 +50,7 @@ def test_next_sync_committee_merkle_proof(spec, state): @spec_state_test def test_finality_root_merkle_proof(spec, state): yield "object", state - finality_branch = spec.compute_merkle_proof_for_state( - state, spec.FINALIZED_ROOT_INDEX) + finality_branch = spec.compute_merkle_proof(state, spec.FINALIZED_ROOT_INDEX) yield "proof", { "leaf": "0x" + state.finalized_checkpoint.root.hex(), "leaf_index": spec.FINALIZED_ROOT_INDEX, diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py index 8d3bf8e3cb..8401a94442 100644 --- a/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py @@ -15,8 +15,7 @@ def test_execution_merkle_proof(spec, state): block = state_transition_with_full_block(spec, state, True, False) yield "object", block.message.body - execution_branch = spec.compute_merkle_proof_for_block_body( - block.message.body, spec.EXECUTION_PAYLOAD_INDEX) + execution_branch = spec.compute_merkle_proof(block.message.body, spec.EXECUTION_PAYLOAD_INDEX) yield "proof", { "leaf": "0x" + block.message.body.execution_payload.hash_tree_root().hex(), "leaf_index": spec.EXECUTION_PAYLOAD_INDEX, diff --git a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py new file mode 100644 index 0000000000..75bcacc7fc --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py @@ -0,0 +1,46 @@ +from eth2spec.test.context import ( + spec_state_test, + with_deneb_and_later, + with_test_suite_name, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, + sign_block +) +from eth2spec.test.helpers.execution_payload import ( + compute_el_block_hash, +) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, +) + + +@with_test_suite_name("BeaconBlockBody") +@with_deneb_and_later +@spec_state_test +def test_blob_kzg_commitment_merkle_proof(spec, state): + opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=1) + block = build_empty_block_for_next_slot(spec, state) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = sign_block(spec, state, block, proposer_index=0) + blob_sidecars = spec.get_blob_sidecars(signed_block, blobs, proofs) + blob_index = 0 + blob_sidecar = blob_sidecars[blob_index] + + yield "object", block.body + kzg_commitment_inclusion_proof = blob_sidecar.kzg_commitment_inclusion_proof + gindex = spec.get_generalized_index(spec.BeaconBlockBody, 'blob_kzg_commitments', blob_index) + yield "proof", { + "leaf": "0x" + blob_sidecar.kzg_commitment.hash_tree_root().hex(), + "leaf_index": gindex, + "branch": ['0x' + root.hex() for root in kzg_commitment_inclusion_proof] + } + assert spec.is_valid_merkle_branch( + leaf=blob_sidecar.kzg_commitment.hash_tree_root(), + branch=blob_sidecar.kzg_commitment_inclusion_proof, + depth=spec.floorlog2(gindex), + index=spec.get_subtree_index(gindex), + root=blob_sidecar.signed_block_header.message.body_root, + ) diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 087bd63c60..f3fa956d0a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -20,3 +20,6 @@ def test_networking(spec): assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK # Start with the same size, but `BLOB_SIDECAR_SUBNET_COUNT` could potentially increase later. assert spec.config.BLOB_SIDECAR_SUBNET_COUNT == spec.MAX_BLOBS_PER_BLOCK + for i in range(spec.MAX_BLOB_COMMITMENTS_PER_BLOCK): + gindex = spec.get_generalized_index(spec.BeaconBlockBody, 'blob_kzg_commitments', i) + assert spec.floorlog2(gindex) == spec.KZG_COMMITMENT_INCLUSION_PROOF_DEPTH diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 876824107a..7ed88cba45 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -1,5 +1,5 @@ +import random from eth2spec.test.context import ( - always_bls, spec_state_test, with_deneb_and_later, ) @@ -10,58 +10,59 @@ get_sample_opaque_tx, ) from eth2spec.test.helpers.block import ( - build_empty_block_for_next_slot -) -from eth2spec.test.helpers.keys import ( - pubkey_to_privkey + build_empty_block_for_next_slot, + sign_block ) -@with_deneb_and_later -@spec_state_test -def test_blob_sidecar_signature(spec, state): - """ - Test `get_blob_sidecar_signature` - """ - blob_count = 4 +def _get_sample_sidecars(spec, state, rng): block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] + + # 2 txs, each has 2 blobs + blob_count = 2 + opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) + assert opaque_tx_1 != opaque_tx_2 + + block.body.blob_kzg_commitments = blob_kzg_commitments_1 + blob_kzg_commitments_2 + block.body.execution_payload.transactions = [opaque_tx_1, opaque_tx_2] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - blob_sidecars = spec.get_blob_sidecars(block, blobs, proofs) - proposer = state.validators[blob_sidecars[1].proposer_index] - privkey = pubkey_to_privkey[proposer.pubkey] - sidecar_signature = spec.get_blob_sidecar_signature(state, - blob_sidecars[1], - privkey) + blobs = blobs_1 + blobs_2 + proofs = proofs_1 + proofs_2 + signed_block = sign_block(spec, state, block, proposer_index=0) + return spec.get_blob_sidecars(signed_block, blobs, proofs) + - signed_blob_sidecar = spec.SignedBlobSidecar(message=blob_sidecars[1], signature=sidecar_signature) +@with_deneb_and_later +@spec_state_test +def test_blob_sidecar_inclusion_proof_correct(spec, state): + rng = random.Random(1234) + blob_sidecars = _get_sample_sidecars(spec, state, rng) - assert spec.verify_blob_sidecar_signature(state, signed_blob_sidecar) + for blob_sidecar in blob_sidecars: + assert spec.verify_blob_sidecar_inclusion_proof(blob_sidecar) @with_deneb_and_later @spec_state_test -@always_bls -def test_blob_sidecar_signature_incorrect(spec, state): - """ - Test `get_blob_sidecar_signature` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) +def test_blob_sidecar_inclusion_proof_incorrect_wrong_body(spec, state): + rng = random.Random(1234) + blob_sidecars = _get_sample_sidecars(spec, state, rng) - blob_sidecars = spec.get_blob_sidecars(block, blobs, proofs) + for blob_sidecar in blob_sidecars: + block = blob_sidecar.signed_block_header.message + block.body_root = spec.hash(block.body_root) # mutate body root to break proof + assert not spec.verify_blob_sidecar_inclusion_proof(blob_sidecar) - sidecar_signature = spec.get_blob_sidecar_signature(state, - blob_sidecars[1], - 123) - signed_blob_sidecar = spec.SignedBlobSidecar(message=blob_sidecars[1], signature=sidecar_signature) +@with_deneb_and_later +@spec_state_test +def test_blob_sidecar_inclusion_proof_incorrect_wrong_proof(spec, state): + rng = random.Random(1234) + blob_sidecars = _get_sample_sidecars(spec, state, rng) - assert not spec.verify_blob_sidecar_signature(state, signed_blob_sidecar) + for blob_sidecar in blob_sidecars: + # wrong proof + blob_sidecar.kzg_commitment_inclusion_proof = spec.compute_merkle_proof(spec.BeaconBlockBody(), 0) + assert not spec.verify_blob_sidecar_inclusion_proof(blob_sidecar) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index ceca145e94..1878832c37 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -56,13 +56,11 @@ def create_update(spec, if with_next: update.next_sync_committee = attested_state.next_sync_committee - update.next_sync_committee_branch = spec.compute_merkle_proof_for_state( - attested_state, spec.NEXT_SYNC_COMMITTEE_INDEX) + update.next_sync_committee_branch = spec.compute_merkle_proof(attested_state, spec.NEXT_SYNC_COMMITTEE_INDEX) if with_finality: update.finalized_header = spec.block_to_light_client_header(finalized_block) - update.finality_branch = spec.compute_merkle_proof_for_state( - attested_state, spec.FINALIZED_ROOT_INDEX) + update.finality_branch = spec.compute_merkle_proof(attested_state, spec.FINALIZED_ROOT_INDEX) update.sync_aggregate, update.signature_slot = get_sync_aggregate( spec, attested_state, num_participants) diff --git a/tests/formats/merkle_proof/README.md b/tests/formats/merkle_proof/README.md new file mode 100644 index 0000000000..77822daabe --- /dev/null +++ b/tests/formats/merkle_proof/README.md @@ -0,0 +1,4 @@ +# Merkle proof tests + +Handlers: +- `single_merkle_proof`: see [Single leaf merkle proof test format](../light_client/single_merkle_proof.md) diff --git a/tests/generators/merkle_proof/README.md b/tests/generators/merkle_proof/README.md new file mode 100644 index 0000000000..fb4d05fda8 --- /dev/null +++ b/tests/generators/merkle_proof/README.md @@ -0,0 +1,5 @@ +# Merkle proof tests + +The purpose of this test-generator is to provide test-vectors for validating the correct implementation of the Merkle proof verification. + +Test-format documentation can be found [here](../../formats/merkle_proof/README.md). diff --git a/tests/generators/merkle_proof/__init__.py b/tests/generators/merkle_proof/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/generators/merkle_proof/main.py b/tests/generators/merkle_proof/main.py new file mode 100644 index 0000000000..59a0e8ce65 --- /dev/null +++ b/tests/generators/merkle_proof/main.py @@ -0,0 +1,14 @@ +from eth2spec.test.helpers.constants import DENEB +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + + +if __name__ == "__main__": + deneb_mods = {key: 'eth2spec.test.deneb.merkle_proof.test_' + key for key in [ + 'single_merkle_proof', + ]} + + all_mods = { + DENEB: deneb_mods, + } + + run_state_test_generators(runner_name="merkle_proof", all_mods=all_mods) diff --git a/tests/generators/merkle_proof/requirements.txt b/tests/generators/merkle_proof/requirements.txt new file mode 100644 index 0000000000..1822486863 --- /dev/null +++ b/tests/generators/merkle_proof/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator]