Phase 1 rebase

Makefile

@@ -104,7 +104,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
 	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@
 
 $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_beacon-chain.md $(SPEC_DIR)/core/1_fraud-proofs.md $(SPEC_DIR)/core/1_phase1-fork.md $@
 
 CURRENT_DIR = ${CURDIR}
 

README.md

@@ -18,9 +18,10 @@ Core specifications for Eth2 client validation can be found in [specs/core](spec
 * [Honest Validator](specs/validator/0_beacon-chain-validator.md)
 
 ### Phase 1
+* [From Phase 0 to Phase 1](specs/core/1_phase1_fork.md)
+* [The Beacon Chain for Shards](specs/core/1_beacon-chain.md)
 * [Custody Game](specs/core/1_custody-game.md)
-* [Shard Data Chains](specs/core/1_shard-data-chains.md)
-* [Misc beacon chain updates](specs/core/1_beacon-chain-misc.md)
+* [Shard Transition and Fraud Proofs](specs/core/1_fraud_proofs.md)
 
 ### Phase 2
 

configs/mainnet.yaml

@@ -142,6 +142,12 @@ DOMAIN_BEACON_ATTESTER: 0x01000000
 DOMAIN_RANDAO: 0x02000000
 DOMAIN_DEPOSIT: 0x03000000
 DOMAIN_VOLUNTARY_EXIT: 0x04000000
-DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000
-DOMAIN_SHARD_PROPOSER: 0x80000000
-DOMAIN_SHARD_ATTESTER: 0x81000000
+DOMAIN_SHARD_PROPOSAL: 0x80000000
+DOMAIN_SHARD_COMMITTEE: 0x81000000
+DOMAIN_LIGHT_CLIENT: 0x82000000
+DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
+
+
+# Phase 1
+# ---------------------------------------------------------------
+INITIAL_ACTIVE_SHARDS: 64

configs/minimal.yaml

@@ -145,16 +145,13 @@ DOMAIN_BEACON_ATTESTER: 0x01000000
 DOMAIN_RANDAO: 0x02000000
 DOMAIN_DEPOSIT: 0x03000000
 DOMAIN_VOLUNTARY_EXIT: 0x04000000
-DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000
-DOMAIN_SHARD_PROPOSER: 0x80000000
-DOMAIN_SHARD_ATTESTER: 0x81000000
+DOMAIN_SHARD_PROPOSAL: 0x80000000
+DOMAIN_SHARD_COMMITTEE: 0x81000000
+DOMAIN_LIGHT_CLIENT: 0x82000000
+DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
 
 
 # Phase 1
 # ---------------------------------------------------------------
-SHARD_SLOTS_PER_BEACON_SLOT: 2
-EPOCHS_PER_SHARD_PERIOD: 4
-# PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2
-PHASE_1_FORK_EPOCH: 8
-# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SLOTS_PER_EPOCH
-PHASE_1_FORK_SLOT: 64
+# [customized] reduced for testing
+INITIAL_ACTIVE_SHARDS: 4

scripts/build_spec.py

@@ -9,9 +9,13 @@
     Optional,
 )
 
+CONFIG_LOADER = '''
+apply_constants_preset(globals())
+'''
 
-PHASE0_IMPORTS = '''from typing import (
-    Any, Dict, Set, Sequence, Tuple, Optional
+PHASE0_IMPORTS = '''from eth2spec.config.apply_config import apply_constants_preset
+from typing import (
+    Any, Callable, Dict, Set, Sequence, Tuple, Optional
 )
 
 from dataclasses import (
@@ -33,8 +37,10 @@
 
 from eth2spec.utils.hash_function import hash
 '''
-PHASE1_IMPORTS = '''from typing import (
-    Any, Dict, Set, Sequence, MutableSequence, NewType, Tuple, Union,
+PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
+from eth2spec.config.apply_config import apply_constants_preset
+from typing import (
+    Any, Callable, Dict, Set, Sequence, NewType, Tuple, Union,
 )
 from math import (
     log2,
@@ -45,15 +51,12 @@
     field,
 )
 
-from eth2spec.utils.ssz.ssz_impl import (
-    hash_tree_root,
-    is_zero,
-)
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 from eth2spec.utils.ssz.ssz_typing import (
     BasicValue, Elements, BaseBytes, BaseList, SSZType,
     Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector, Bits,
     Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96,
-    uint64, bit, boolean, byte,
+    uint64, uint8, bit, boolean,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
@@ -101,51 +104,15 @@ def compute_committee(indices: Sequence[ValidatorIndex],  # type: ignore
 
     if param_hash not in committee_cache:
         committee_cache[param_hash] = _compute_committee(indices, seed, index, count)
-    return committee_cache[param_hash]
-
-
-# Access to overwrite spec constants based on configuration
-def apply_constants_preset(preset: Dict[str, Any]) -> None:
-    global_vars = globals()
-    for k, v in preset.items():
-        if k.startswith('DOMAIN_'):
-            global_vars[k] = DomainType(v)  # domain types are defined as bytes in the configs
-        else:
-            global_vars[k] = v
-
-    # Deal with derived constants
-    global_vars['GENESIS_EPOCH'] = compute_epoch_at_slot(GENESIS_SLOT)
-
-    # Initialize SSZ types again, to account for changed lengths
-    init_SSZ_types()
-'''
-
-
-def remove_for_phase1(functions: Dict[str, str]):
-    for key, value in functions.items():
-        lines = value.split("\n")
-        lines = filter(lambda s: "[to be removed in phase 1]" not in s, lines)
-        functions[key] = "\n".join(lines)
-
-
-def strip_comments(raw: str) -> str:
-    comment_line_regex = re.compile(r'^\s+# ')
-    lines = raw.split('\n')
-    out = []
-    for line in lines:
-        if not comment_line_regex.match(line):
-            if '  #' in line:
-                line = line[:line.index('  #')]
-            out.append(line)
-    return '\n'.join(out)
+    return committee_cache[param_hash]'''
 
 
 def objects_to_spec(functions: Dict[str, str],
                     custom_types: Dict[str, str],
                     constants: Dict[str, str],
                     ssz_objects: Dict[str, str],
-                    inserts: Dict[str, str],
                     imports: Dict[str, str],
+                    version: str,
                     ) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
@@ -167,27 +134,18 @@ def objects_to_spec(functions: Dict[str, str],
             constants[k] += "  # noqa: E501"
     constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants))
     ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values())
-    ssz_objects_reinitialization_spec = (
-        'def init_SSZ_types() -> None:\n    global_vars = globals()\n\n    '
-        + '\n\n    '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n    ', value[:-1]))
-                           for value in ssz_objects.values()])
-        + '\n\n'
-        + '\n'.join(map(lambda x: '    global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys()))
-    )
     spec = (
         imports
+        + '\n\n' + f"version = \'{version}\'\n"
         + '\n\n' + new_type_definitions
         + '\n' + SUNDRY_CONSTANTS_FUNCTIONS
         + '\n\n' + constants_spec
-        + '\n\n\n' + ssz_objects_instantiation_spec
+        + '\n\n' + CONFIG_LOADER
+        + '\n\n' + ssz_objects_instantiation_spec
         + '\n\n' + functions_spec
         + '\n' + SUNDRY_FUNCTIONS
-        + '\n\n' + ssz_objects_reinitialization_spec
         + '\n'
     )
-    # Handle @inserts
-    for key, value in inserts.items():
-        spec = re.sub('[ ]*# %s\\n' % key, value, spec)
     return spec
 
 
@@ -204,10 +162,10 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
 
 
 ignored_dependencies = [
-    'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'ByteList', 'ByteVector'
+    'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature',
     'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
-    'bytes', 'byte', 'ByteVector'  # to be removed after updating spec doc
+    'bytes', 'byte', 'ByteList', 'ByteVector'
 ]
 
 
@@ -240,32 +198,26 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]
     and returns the newer versions of the objects in dependency order.
     """
     for key, value in new_objects.items():
-        if key in old_objects:
-            # remove trailing newline
-            old_objects[key] = old_objects[key]
-            # remove leading variable name
-            value = re.sub(r'^class [\w]*\(Container\):\n', '', value)
-        old_objects[key] = old_objects.get(key, '') + value
-    dependency_order_ssz_objects(old_objects, custom_types)
+        old_objects[key] = value
     return old_objects
 
 
-# inserts are handled the same way as functions
-combine_inserts = combine_functions
-
-
 def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     """
     Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function.
     """
-    functions0, custom_types0, constants0, ssz_objects0, inserts0 = spec0
-    functions1, custom_types1, constants1, ssz_objects1, inserts1 = spec1
+    functions0, custom_types0, constants0, ssz_objects0 = spec0
+    functions1, custom_types1, constants1, ssz_objects1 = spec1
     functions = combine_functions(functions0, functions1)
     custom_types = combine_constants(custom_types0, custom_types1)
     constants = combine_constants(constants0, constants1)
     ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types)
-    inserts = combine_inserts(inserts0, inserts1)
-    return functions, custom_types, constants, ssz_objects, inserts
+    return functions, custom_types, constants, ssz_objects
+
+
+def dependency_order_spec(objs: SpecObject):
+    functions, custom_types, constants, ssz_objects = objs
+    dependency_order_ssz_objects(ssz_objects, custom_types)
 
 
 def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
@@ -276,7 +228,8 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
     spec_objects = phase0_spec
     for value in [fork_choice_spec, v_guide]:
         spec_objects = combine_spec_objects(spec_objects, value)
-    spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS)
+    dependency_order_spec(spec_objects)
+    spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS, 'phase0')
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)
@@ -287,24 +240,25 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
                       phase0_fork_choice_sourcefile: str,
                       merkle_proofs_sourcefile: str,
                       phase1_custody_sourcefile: str,
-                      phase1_shard_sourcefile: str,
-                      phase1_beacon_misc_sourcefile: str,
+                      phase1_beacon_sourcefile: str,
+                      phase1_fraud_sourcefile: str,
+                      phase1_fork_sourcefile: str,
                       outfile: str=None) -> Optional[str]:
     all_sourcefiles = (
         phase0_beacon_sourcefile,
         phase0_fork_choice_sourcefile,
         merkle_proofs_sourcefile,
         phase1_custody_sourcefile,
-        phase1_shard_sourcefile,
-        phase1_beacon_misc_sourcefile,
+        phase1_beacon_sourcefile,
+        phase1_fraud_sourcefile,
+        phase1_fork_sourcefile,
     )
     all_spescs = [get_spec(spec) for spec in all_sourcefiles]
-    for spec in all_spescs:
-        remove_for_phase1(spec[0])
     spec_objects = all_spescs[0]
     for value in all_spescs[1:]:
         spec_objects = combine_spec_objects(spec_objects, value)
-    spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS)
+    dependency_order_spec(spec_objects)
+    spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, 'phase1')
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)
@@ -325,9 +279,10 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
     2nd argument is input /core/0_fork-choice.md
     3rd argument is input /light_client/merkle_proofs.md
     4th argument is input /core/1_custody-game.md
-    5th argument is input /core/1_shard-data-chains.md
-    6th argument is input /core/1_beacon-chain-misc.md
-    7th argument is output spec.py
+    5th argument is input /core/1_beacon-chain.md
+    6th argument is input /core/1_fraud-proofs.md
+    7th argument is input /core/1_phase1-fork.md
+    8th argument is output spec.py
 '''
     parser = ArgumentParser(description=description)
     parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
@@ -340,14 +295,14 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
         else:
             print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.")
     elif args.phase == 1:
-        if len(args.files) == 7:
+        if len(args.files) == 8:
             build_phase1_spec(*args.files)
         else:
             print(
                 " Phase 1 requires input files as well as an output file:\n"
                 "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n"
                 "\t light_client: (merkle_proofs.md)\n"
-                "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n"
+                "\t core/phase_1: (1_custody-game.md, 1_beacon-chain.md, 1_fraud-proofs.md, 1_phase1-fork.md)\n"
                 "\t and output.py"
             )
     else:

scripts/function_puller.py

@@ -3,8 +3,6 @@
 
 
 FUNCTION_REGEX = r'^def [\w_]*'
-BEGIN_INSERT_REGEX = r'# begin insert '
-END_INSERT_REGEX = r'# end insert'
 
 SpecObject = NewType('SpecObjects', Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]])
 
@@ -15,22 +13,18 @@ def get_spec(file_name: str) -> SpecObject:
     functions = {function_name: function_code}
     constants= {constant_name: constant_code}
     ssz_objects= {object_name: object}
-    inserts= {insert_tag: code to be inserted}
 
     Note: This function makes heavy use of the inherent ordering of dicts,
     if this is not supported by your python version, it will not work.
     """
     pulling_from = None  # line number of start of latest object
     current_name = None  # most recent section title
-    insert_name = None  # stores the label of the current insert object
-    functions = {}
-    constants = {}
-    ssz_objects = {}
-    inserts = {}
+    functions: Dict[str, str] = {}
+    constants: Dict[str, str] = {}
+    ssz_objects: Dict[str, str] = {}
     function_matcher = re.compile(FUNCTION_REGEX)
-    inserts_matcher = re.compile(BEGIN_INSERT_REGEX)
     is_ssz = False
-    custom_types = {}
+    custom_types: Dict[str, str] = {}
     for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()
         if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
@@ -40,15 +34,6 @@ def get_spec(file_name: str) -> SpecObject:
             pulling_from = linenum + 1
         elif line[:3] == '```':
             pulling_from = None
-        elif inserts_matcher.match(line) is not None:
-            # Find @insert names
-            insert_name = re.search(r'@[\w]*', line).group(0)
-        elif insert_name is not None:
-            # In insert mode, either the next line is more code, or the end of the insert
-            if re.match(END_INSERT_REGEX, line) is not None:
-                insert_name = None
-            else:
-                inserts[insert_name] = inserts.get(insert_name, '') + line + '\n'
         else:
             # Handle function definitions & ssz_objects
             if pulling_from is not None:
@@ -84,4 +69,4 @@ def get_spec(file_name: str) -> SpecObject:
                         constants[row[0]] = row[1].replace('**TBD**', '2**32')
                     elif row[1].startswith('uint') or row[1].startswith('Bytes'):
                         custom_types[row[0]] = row[1]
-    return functions, custom_types, constants, ssz_objects, inserts
+    return SpecObject((functions, custom_types, constants, ssz_objects))