Add blob sidecar inclusion proof

presets/mainnet/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17

presets/minimal/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# [customized] `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9

pysetup/helpers.py

@@ -111,8 +111,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         return out
 
     # Merge all constant objects
-    hardcoded_ssz_dep_constants =         reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()},                    builders, {})
+    hardcoded_ssz_dep_constants =         reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
     hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
+    hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
     # Concatenate all strings
     imports =              reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name)  ).strip("\n"), builders, "")
     preparations =         reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations()        ).strip("\n"), builders, "")
@@ -126,6 +127,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
     ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
     custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
+    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
     spec_strs = [
         imports,
         preparations,
@@ -147,6 +149,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
         # as same as the spec definition.
         ssz_dep_constants_verification,
+        func_dep_presets_verification,
     ]
     return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) +  "\n"
 
@@ -223,6 +226,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
     config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
     ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
+    func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets)
     ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types)
     dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses)
     return SpecObject(
@@ -233,6 +237,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,
+        func_dep_presets=func_dep_presets,
         ssz_objects=ssz_objects,
         dataclasses=dataclasses,
     )

pysetup/spec_builders/base.py

@@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:  #
         """
         return {}
 
+    @classmethod
+    def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+        return {}
+
     @classmethod
     def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
         return functions

pysetup/spec_builders/deneb.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from .base import BaseSpecBuilder
 from ..constants import DENEB
 
@@ -23,7 +24,12 @@ def sundry_functions(cls) -> str:
         return '''
 def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
     # pylint: disable=unused-argument
-    return [], []'''
+    return [], []
+
+
+def compute_commitment_inclusion_proof(body: BeaconBlockBody, index: GeneralizedIndex) -> Sequence[Bytes32]:
+    return build_proof(body.get_backing(), index)
+'''
 
     @classmethod
     def execution_engine_cls(cls) -> str:
@@ -63,9 +69,22 @@ def verify_and_notify_new_payload(self: ExecutionEngine,
 
 
     @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
         return {
             'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
             'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+            'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
+        }
+
+    @classmethod
+    def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
+        return {
+            'BLOB_KZG_COMMITMENTS_GINDEX': 'GeneralizedIndex(27)',
+        }
+
+    @classmethod
+    def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+        return {
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value,
         }

pysetup/typing.py

@@ -22,6 +22,7 @@ class SpecObject(NamedTuple):
     preset_vars: Dict[str, VariableDefinition]
     config_vars: Dict[str, VariableDefinition]
     ssz_dep_constants: Dict[str, str]  # the constants that depend on ssz_objects
+    func_dep_presets: Dict[str, str]  # the constants that depend on functions
     ssz_objects: Dict[str, str]
     dataclasses: Dict[str, str]
 

setup.py

@@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
     preset_vars: Dict[str, VariableDefinition] = {}
     config_vars: Dict[str, VariableDefinition] = {}
     ssz_dep_constants: Dict[str, str] = {}
+    func_dep_presets: Dict[str, str] = {}
     ssz_objects: Dict[str, str] = {}
     dataclasses: Dict[str, str] = {}
     custom_types: Dict[str, str] = {}
@@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
                     value_cell = cells[1]
                     value = value_cell.children[0].children
+
+                    description = None
+                    if len(cells) >= 3:
+                        description_cell = cells[2]
+                        if len(description_cell.children) > 0:
+                            description = description_cell.children[0].children
+                            if isinstance(description, list):
+                                # marko parses `**X**` as a list containing a X
+                                description = description[0].children
+
                     if isinstance(value, list):
                         # marko parses `**X**` as a list containing a X
                         value = value[0].children
@@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                         ssz_dep_constants[name] = value
                         continue
 
+                    if description is not None and description.startswith("<!-- predefined -->"):
+                        func_dep_presets[name] = value
+
                     value_def = _parse_value(name, value)
                     if name in preset:
                         preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None)
@@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,
+        func_dep_presets=func_dep_presets,
         ssz_objects=ssz_objects,
         dataclasses=dataclasses,
     )

specs/deneb/beacon-chain.md

@@ -25,6 +25,7 @@
 - [Helper functions](#helper-functions)
   - [Misc](#misc)
     - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash)
+    - [`is_valid_merkle_path`](#is_valid_merkle_path)
   - [Beacon state accessors](#beacon-state-accessors)
     - [Modified `get_attestation_participation_flag_indices`](#modified-get_attestation_participation_flag_indices)
     - [New `get_validator_activation_churn_limit`](#new-get_validator_activation_churn_limit)
@@ -187,6 +188,19 @@ def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> Versioned
     return VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:]
 ```
 
+#### `is_valid_merkle_path`
+
+```python
+def is_valid_merkle_path(leaf: Bytes32, branch: Sequence[Bytes32], gindex: int, root: Root) -> bool:
+    value = leaf
+    for i in range(len(branch)):
+        if (gindex >> i) & 1 == 0:
+            value = hash(branch[i] + value)
+        else:
+            value = hash(value + branch[i])
+    return value == root
+```
+
 ### Beacon state accessors
 
 #### Modified `get_attestation_participation_flag_indices`

specs/deneb/p2p-interface.md

@@ -11,13 +11,14 @@ The specification of these changes continues in the same format as the network s
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
 - [Modifications in Deneb](#modifications-in-deneb)
+  - [Constant](#constant)
+  - [Preset](#preset)
   - [Configuration](#configuration)
   - [Containers](#containers)
     - [`BlobSidecar`](#blobsidecar)
-    - [`SignedBlobSidecar`](#signedblobsidecar)
     - [`BlobIdentifier`](#blobidentifier)
     - [Helpers](#helpers)
-      - [`verify_blob_sidecar_signature`](#verify_blob_sidecar_signature)
+      - [`verify_blob_sidecar_inclusion_proof`](#verify_blob_sidecar_inclusion_proof)
   - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
     - [Topics and messages](#topics-and-messages)
       - [Global topics](#global-topics)
@@ -41,6 +42,22 @@ The specification of these changes continues in the same format as the network s
 
 ## Modifications in Deneb
 
+### Constant
+
+*[New in Deneb:EIP4844]*
+
+| Name                                     | Value                             | Description                                                         |
+|------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
+| `BLOB_KZG_COMMITMENTS_GINDEX`            | `get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')` (= 27) | `blob_kzg_commitments` field gindex on `BeaconBlockBody` container |
+
+### Preset
+
+*[New in Deneb:EIP4844]*
+
+| Name                                     | Value                             | Description                                                         |
+|------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
+| `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH`   | `uint64(floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK))` (= 17) | <!-- predefined --> Merkle proof depth for `blob_kzg_commitments` list item |
+
 ### Configuration
 
 *[New in Deneb:EIP4844]*
@@ -60,24 +77,12 @@ The specification of these changes continues in the same format as the network s
 
 ```python
 class BlobSidecar(Container):
-    block_root: Root
     index: BlobIndex  # Index of blob in block
-    slot: Slot
-    block_parent_root: Root  # Proposer shuffling determinant
-    proposer_index: ValidatorIndex
     blob: Blob
     kzg_commitment: KZGCommitment
     kzg_proof: KZGProof  # Allows for quick verification of kzg_commitment
-```
-
-#### `SignedBlobSidecar`
-
-*[New in Deneb:EIP4844]*
-
-```python
-class SignedBlobSidecar(Container):
-    message: BlobSidecar
-    signature: BLSSignature
+    signed_block_header: SignedBeaconBlockHeader
+    commitment_inclusion_proof: List[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH]
 ```
 
 #### `BlobIdentifier`
@@ -92,13 +97,18 @@ class BlobIdentifier(Container):
 
 #### Helpers
 
-##### `verify_blob_sidecar_signature`
+##### `verify_blob_sidecar_inclusion_proof`
 
 ```python
-def verify_blob_sidecar_signature(state: BeaconState, signed_blob_sidecar: SignedBlobSidecar) -> bool:
-    proposer = state.validators[signed_blob_sidecar.message.proposer_index]
-    signing_root = compute_signing_root(signed_blob_sidecar.message, get_domain(state, DOMAIN_BLOB_SIDECAR))
-    return bls.Verify(proposer.pubkey, signing_root, signed_blob_sidecar.signature)
+def verify_blob_sidecar_inclusion_proof(blob_sidecar: BlobSidecar) -> bool:
+    gindex = get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', blob_sidecar.index)
+    return is_valid_merkle_branch(
+        leaf=blob_sidecar.kzg_commitment.hash_tree_root(),
+        branch=blob_sidecar.commitment_inclusion_proof,
+        depth=floorlog2(gindex),
+        index=get_subtree_index(gindex),
+        root=blob_sidecar.signed_block_header.message.body_root,
+    )
 ```
 
 ### The gossip domain: gossipsub
@@ -123,7 +133,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar
 
 | Name | Message Type |
 | - | - |
-| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` [New in Deneb:EIP4844] |
+| `blob_sidecar_{subnet_id}` | `BlobSidecar` [New in Deneb:EIP4844] |
 
 ##### Global topics
 
@@ -146,18 +156,19 @@ New validation:
 
 This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`.
 
-The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`:
+The following validations MUST pass before forwarding the `blob_sidecar` on the network, assuming the alias `sidecar = blob_sidecar` and `block_header = blob_sidecar.signed_block_header.message`:
 
 - _[REJECT]_ The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` -- i.e. `sidecar.index < MAX_BLOBS_PER_BLOCK`.
 - _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(sidecar.index) == subnet_id`.
-- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
-- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
-- _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
-- _[REJECT]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) passes validation.
-- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `sidecar.block_parent_root`).
-- _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid as verified by `verify_blob_sidecar_signature`.
-- _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple `(sidecar.block_root, sidecar.index)`.
-- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_parent_root`/`slot`).
+- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `block_header.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
+- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `block_header.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
+- _[IGNORE]_ The sidecar's block's parent (defined by `block_header.parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
+- _[REJECT]_ The sidecar's block's parent (defined by `block_header.parent_root`) passes validation.
+- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `block_header.parent_root`).
+- _[REJECT]_ The proposer signature of `blob_sidecar.signed_block_header`, is valid with respect to the `block_header.proposer_index` pubkey.
+- _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof`.
+- _[IGNORE]_ The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, sidecar.index) with valid header signature and sidecar inclusion proof
+- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`).
   If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message.
 
 ###### `beacon_aggregate_and_proof`