Skip to content

Commit

Permalink
Merge pull request #3531 from dapplion/blob-p2p-proof
Browse files Browse the repository at this point in the history
Add blob sidecar inclusion proof
  • Loading branch information
djrtwo authored Nov 2, 2023
2 parents 38d354f + 7118c30 commit c6f7adf
Show file tree
Hide file tree
Showing 27 changed files with 243 additions and 150 deletions.
2 changes: 2 additions & 0 deletions presets/mainnet/deneb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
# `uint64(6)`
MAX_BLOBS_PER_BLOCK: 6
# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17
2 changes: 2 additions & 0 deletions presets/minimal/deneb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
# `uint64(6)`
MAX_BLOBS_PER_BLOCK: 6
# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9
10 changes: 7 additions & 3 deletions pysetup/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
if k in [
"ceillog2",
"floorlog2",
"compute_merkle_proof_for_block_body",
"compute_merkle_proof_for_state",
"compute_merkle_proof",
]:
del spec_object.functions[k]

Expand Down Expand Up @@ -111,8 +110,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
return out

# Merge all constant objects
hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
# Concatenate all strings
imports = reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name) ).strip("\n"), builders, "")
preparations = reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations() ).strip("\n"), builders, "")
Expand All @@ -126,6 +126,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
spec_strs = [
imports,
preparations,
Expand All @@ -147,6 +148,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
# Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
# as same as the spec definition.
ssz_dep_constants_verification,
func_dep_presets_verification,
]
return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n"

Expand Down Expand Up @@ -223,6 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets)
ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types)
dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses)
return SpecObject(
Expand All @@ -233,6 +236,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
preset_vars=preset_vars,
config_vars=config_vars,
ssz_dep_constants=ssz_dep_constants,
func_dep_presets=func_dep_presets,
ssz_objects=ssz_objects,
dataclasses=dataclasses,
)
Expand Down
8 changes: 4 additions & 4 deletions pysetup/spec_builders/altair.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,16 @@ def preparations(cls):
@classmethod
def sundry_functions(cls) -> str:
return '''
def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex:
def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex:
ssz_path = Path(ssz_class)
for item in path:
ssz_path = ssz_path / item
return GeneralizedIndex(ssz_path.gindex())
def compute_merkle_proof_for_state(state: BeaconState,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(state.get_backing(), index)'''
def compute_merkle_proof(object: SSZObject,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(object.get_backing(), index)'''


@classmethod
Expand Down
4 changes: 4 additions & 0 deletions pysetup/spec_builders/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: #
"""
return {}

@classmethod
def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
return {}

@classmethod
def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
return functions
9 changes: 0 additions & 9 deletions pysetup/spec_builders/capella.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,6 @@ def imports(cls, preset_name: str):
from eth2spec.bellatrix import {preset_name} as bellatrix
'''


@classmethod
def sundry_functions(cls) -> str:
return '''
def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(body.get_backing(), index)'''


@classmethod
def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
return {
Expand Down
13 changes: 11 additions & 2 deletions pysetup/spec_builders/deneb.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Dict
from .base import BaseSpecBuilder
from ..constants import DENEB

Expand All @@ -23,7 +24,8 @@ def sundry_functions(cls) -> str:
return '''
def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
# pylint: disable=unused-argument
return [], []'''
return [], []
'''

@classmethod
def execution_engine_cls(cls) -> str:
Expand Down Expand Up @@ -63,9 +65,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine,


@classmethod
def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
return {
'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
}

@classmethod
def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
return {
'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value,
}
1 change: 1 addition & 0 deletions pysetup/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class SpecObject(NamedTuple):
preset_vars: Dict[str, VariableDefinition]
config_vars: Dict[str, VariableDefinition]
ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects
func_dep_presets: Dict[str, str] # the constants that depend on functions
ssz_objects: Dict[str, str]
dataclasses: Dict[str, str]

Expand Down
15 changes: 15 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
preset_vars: Dict[str, VariableDefinition] = {}
config_vars: Dict[str, VariableDefinition] = {}
ssz_dep_constants: Dict[str, str] = {}
func_dep_presets: Dict[str, str] = {}
ssz_objects: Dict[str, str] = {}
dataclasses: Dict[str, str] = {}
custom_types: Dict[str, str] = {}
Expand Down Expand Up @@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr

value_cell = cells[1]
value = value_cell.children[0].children

description = None
if len(cells) >= 3:
description_cell = cells[2]
if len(description_cell.children) > 0:
description = description_cell.children[0].children
if isinstance(description, list):
# marko parses `**X**` as a list containing a X
description = description[0].children

if isinstance(value, list):
# marko parses `**X**` as a list containing a X
value = value[0].children
Expand All @@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
ssz_dep_constants[name] = value
continue

if description is not None and description.startswith("<!-- predefined -->"):
func_dep_presets[name] = value

value_def = _parse_value(name, value)
if name in preset:
preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None)
Expand Down Expand Up @@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
preset_vars=preset_vars,
config_vars=config_vars,
ssz_dep_constants=ssz_dep_constants,
func_dep_presets=func_dep_presets,
ssz_objects=ssz_objects,
dataclasses=dataclasses,
)
Expand Down
18 changes: 9 additions & 9 deletions specs/altair/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

- [Introduction](#introduction)
- [Helper functions](#helper-functions)
- [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
- [`compute_merkle_proof`](#compute_merkle_proof)
- [`block_to_light_client_header`](#block_to_light_client_header)
- [Deriving light client data](#deriving-light-client-data)
- [`create_light_client_bootstrap`](#create_light_client_bootstrap)
Expand All @@ -27,11 +27,13 @@ This document provides helper functions to enable full nodes to serve light clie

## Helper functions

### `compute_merkle_proof_for_state`
### `compute_merkle_proof`

This function return the Merkle proof of the given SSZ object `object` at generalized index `index`.

```python
def compute_merkle_proof_for_state(state: BeaconState,
index: GeneralizedIndex) -> Sequence[Bytes32]:
def compute_merkle_proof(object: SSZObject,
index: GeneralizedIndex) -> Sequence[Bytes32]:
...
```

Expand Down Expand Up @@ -73,7 +75,7 @@ def create_light_client_bootstrap(state: BeaconState,
return LightClientBootstrap(
header=block_to_light_client_header(block),
current_sync_committee=state.current_sync_committee,
current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX),
current_sync_committee_branch=compute_merkle_proof(state, CURRENT_SYNC_COMMITTEE_INDEX),
)
```

Expand Down Expand Up @@ -120,8 +122,7 @@ def create_light_client_update(state: BeaconState,
# `next_sync_committee` is only useful if the message is signed by the current sync committee
if update_attested_period == update_signature_period:
update.next_sync_committee = attested_state.next_sync_committee
update.next_sync_committee_branch = compute_merkle_proof_for_state(
attested_state, NEXT_SYNC_COMMITTEE_INDEX)
update.next_sync_committee_branch = compute_merkle_proof(attested_state, NEXT_SYNC_COMMITTEE_INDEX)

# Indicate finality whenever possible
if finalized_block is not None:
Expand All @@ -130,8 +131,7 @@ def create_light_client_update(state: BeaconState,
assert hash_tree_root(update.finalized_header.beacon) == attested_state.finalized_checkpoint.root
else:
assert attested_state.finalized_checkpoint.root == Bytes32()
update.finality_branch = compute_merkle_proof_for_state(
attested_state, FINALIZED_ROOT_INDEX)
update.finality_branch = compute_merkle_proof(attested_state, FINALIZED_ROOT_INDEX)

update.sync_aggregate = block.message.body.sync_aggregate
update.signature_slot = block.message.slot
Expand Down
11 changes: 1 addition & 10 deletions specs/capella/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

- [Introduction](#introduction)
- [Helper functions](#helper-functions)
- [`compute_merkle_proof_for_block_body`](#compute_merkle_proof_for_block_body)
- [Modified `block_to_light_client_header`](#modified-block_to_light_client_header)

<!-- END doctoc generated TOC please keep comment here to allow auto update -->
Expand All @@ -22,14 +21,6 @@ This upgrade adds information about the execution payload to light client data a

## Helper functions

### `compute_merkle_proof_for_block_body`

```python
def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
index: GeneralizedIndex) -> Sequence[Bytes32]:
...
```

### Modified `block_to_light_client_header`

```python
Expand All @@ -55,7 +46,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
transactions_root=hash_tree_root(payload.transactions),
withdrawals_root=hash_tree_root(payload.withdrawals),
)
execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
else:
# Note that during fork transitions, `finalized_header` may still point to earlier forks.
# While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),
Expand Down
2 changes: 1 addition & 1 deletion specs/deneb/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
execution_header.blob_gas_used = payload.blob_gas_used
execution_header.excess_blob_gas = payload.excess_blob_gas

execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
else:
# Note that during fork transitions, `finalized_header` may still point to earlier forks.
# While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),
Expand Down
Loading

0 comments on commit c6f7adf

Please sign in to comment.