diff --git a/beacon-chain/BUILD.bazel b/beacon-chain/BUILD.bazel index 5c84e06e2ed7..9347910afbd2 100644 --- a/beacon-chain/BUILD.bazel +++ b/beacon-chain/BUILD.bazel @@ -59,6 +59,7 @@ go_image( "//shared/featureconfig:go_default_library", "//shared/logutil:go_default_library", "//shared/version:go_default_library", + "//shared/params:go_default_library", "@com_github_ethereum_go_ethereum//log:go_default_library", "@com_github_ipfs_go_log//:go_default_library", "@com_github_joonix_log//:go_default_library", @@ -68,6 +69,7 @@ go_image( "@in_gopkg_urfave_cli_v2//:go_default_library", "@in_gopkg_urfave_cli_v2//altsrc:go_default_library", "@org_uber_go_automaxprocs//:go_default_library", + "@in_gopkg_yaml_v2//:go_default_library", ], ) diff --git a/beacon-chain/main.go b/beacon-chain/main.go index bf4cde9efd71..fdcee2dfc9d2 100644 --- a/beacon-chain/main.go +++ b/beacon-chain/main.go @@ -87,6 +87,7 @@ var appFlags = []cli.Flag{ cmd.LogFileName, cmd.EnableUPnPFlag, cmd.ConfigFileFlag, + cmd.ChainConfigFileFlag, } func init() { diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index acfa3a42f9ce..d603039f312e 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -93,6 +93,11 @@ func NewBeaconNode(cliCtx *cli.Context) (*BeaconNode, error) { return nil, err } + if cliCtx.IsSet(cmd.ChainConfigFileFlag.Name) { + chainConfigFileName := cliCtx.String(cmd.ChainConfigFileFlag.Name) + params.LoadChainConfigFile(chainConfigFileName) + } + featureconfig.ConfigureBeaconChain(cliCtx) flags.ConfigureGlobalFlags(cliCtx) registry := shared.NewServiceRegistry() diff --git a/beacon-chain/rpc/beacon/attestations_test.go b/beacon-chain/rpc/beacon/attestations_test.go index d8db832bc2d4..149dbe5466a0 100644 --- a/beacon-chain/rpc/beacon/attestations_test.go +++ b/beacon-chain/rpc/beacon/attestations_test.go @@ -13,7 +13,6 @@ import ( "github.com/gogo/protobuf/proto" ptypes "github.com/gogo/protobuf/types" "github.com/golang/mock/gomock" - "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-bitfield" mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing" @@ -27,6 +26,7 @@ import ( mockRPC "github.com/prysmaticlabs/prysm/beacon-chain/rpc/testing" stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen" + "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/attestationutil" "github.com/prysmaticlabs/prysm/shared/bytesutil" diff --git a/beacon-chain/rpc/validator/proposer_test.go b/beacon-chain/rpc/validator/proposer_test.go index 72dd8c6c7629..5357fbd0336f 100644 --- a/beacon-chain/rpc/validator/proposer_test.go +++ b/beacon-chain/rpc/validator/proposer_test.go @@ -23,6 +23,7 @@ import ( mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing" beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state" "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen" + "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" mockSync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync/testing" dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db" pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" @@ -33,7 +34,6 @@ import ( "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/prysmaticlabs/prysm/shared/trieutil" - "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" ) func TestGetBlock_OK(t *testing.T) { diff --git a/beacon-chain/usage.go b/beacon-chain/usage.go index 33473af8b22a..cb9f7744dacf 100644 --- a/beacon-chain/usage.go +++ b/beacon-chain/usage.go @@ -61,6 +61,7 @@ var appHelpFlagGroups = []flagGroup{ cmd.ForceClearDB, cmd.ClearDB, cmd.ConfigFileFlag, + cmd.ChainConfigFileFlag, }, }, { diff --git a/fuzz/block_fuzz.go b/fuzz/block_fuzz.go index 89f1cace4dab..09c9b2a2aae0 100644 --- a/fuzz/block_fuzz.go +++ b/fuzz/block_fuzz.go @@ -1,4 +1,3 @@ - package fuzz import ( @@ -33,4 +32,3 @@ func BeaconFuzzBlock(b []byte) ([]byte, bool) { } return success(post) } - diff --git a/fuzz/block_header_fuzz.go b/fuzz/block_header_fuzz.go index fd418ee3334f..58cc1a6736b0 100644 --- a/fuzz/block_header_fuzz.go +++ b/fuzz/block_header_fuzz.go @@ -1,11 +1,11 @@ package fuzz import ( + "github.com/prysmaticlabs/go-ssz" "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks" stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" prylabs_testing "github.com/prysmaticlabs/prysm/fuzz/testing" "github.com/prysmaticlabs/prysm/shared/params" - "github.com/prysmaticlabs/go-ssz" ) // BeaconFuzzBlockHeader using the corpora from sigp/beacon-fuzz. diff --git a/fuzz/common.go b/fuzz/common.go index a678428d043b..1f34cbe66d54 100644 --- a/fuzz/common.go +++ b/fuzz/common.go @@ -38,4 +38,4 @@ func success(post *stateTrie.BeaconState) ([]byte, bool) { panic(err) } return result, true -} \ No newline at end of file +} diff --git a/shared/cmd/flags.go b/shared/cmd/flags.go index eb03b5caf4ed..19bfd2f732d3 100644 --- a/shared/cmd/flags.go +++ b/shared/cmd/flags.go @@ -174,4 +174,9 @@ var ( Name: "config-file", Usage: "The filepath to a yaml file with flag values", } + // ChainConfigFileFlag specifies the filepath to load flag values. + ChainConfigFileFlag = &cli.StringFlag{ + Name: "chain-config-file", + Usage: "The path to a YAML file with chain config values", + } ) diff --git a/shared/params/BUILD.bazel b/shared/params/BUILD.bazel index 8a0fb741b6ef..ef2940a1a0a5 100644 --- a/shared/params/BUILD.bazel +++ b/shared/params/BUILD.bazel @@ -5,6 +5,7 @@ go_library( name = "go_default_library", srcs = [ "config.go", + "loader.go", "network_config.go", ], importpath = "github.com/prysmaticlabs/prysm/shared/params", @@ -12,12 +13,22 @@ go_library( deps = [ "//shared/bytesutil:go_default_library", "@com_github_mohae_deepcopy//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@in_gopkg_yaml_v2//:go_default_library", ], ) go_test( name = "go_default_test", size = "small", - srcs = ["config_test.go"], + srcs = [ + "config_test.go", + "loader_test.go", + ], + data = glob(["*.yaml"]) + [ + "@eth2_spec_tests_mainnet//:test_data", + "@eth2_spec_tests_minimal//:test_data", + ], embed = [":go_default_library"], + deps = ["@io_bazel_rules_go//go/tools/bazel:go_default_library"], ) diff --git a/shared/params/config.go b/shared/params/config.go index 50b23c1df2e5..fac734657000 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -55,7 +55,7 @@ type BeaconChainConfig struct { MinValidatorWithdrawabilityDelay uint64 `yaml:"MIN_VALIDATOR_WITHDRAWABILITY_DELAY"` // MinValidatorWithdrawabilityDelay is the shortest amount of time a validator has to wait to withdraw. PersistentCommitteePeriod uint64 `yaml:"PERSISTENT_COMMITTEE_PERIOD"` // PersistentCommitteePeriod is the minimum amount of epochs a validator must participate before exiting. MinEpochsToInactivityPenalty uint64 `yaml:"MIN_EPOCHS_TO_INACTIVITY_PENALTY"` // MinEpochsToInactivityPenalty defines the minimum amount of epochs since finality to begin penalizing inactivity. - Eth1FollowDistance uint64 `yaml:"EPOCHS_PER_ETH1_VOTING_PERIOD"` // Eth1FollowDistance is the number of eth1.0 blocks to wait before considering a new deposit for voting. This only applies after the chain as been started. + Eth1FollowDistance uint64 // Eth1FollowDistance is the number of eth1.0 blocks to wait before considering a new deposit for voting. This only applies after the chain as been started. SafeSlotsToUpdateJustified uint64 // SafeSlotsToUpdateJustified is the minimal slots needed to update justified check point. SecondsPerETH1Block uint64 `yaml:"SECONDS_PER_ETH1_BLOCK"` // SecondsPerETH1Block is the approximate time for a single eth1 block to be produced. // State list lengths diff --git a/shared/params/loader.go b/shared/params/loader.go new file mode 100644 index 000000000000..482708f9592f --- /dev/null +++ b/shared/params/loader.go @@ -0,0 +1,118 @@ +package params + +import ( + "encoding/hex" + "io/ioutil" + "strings" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// LoadChainConfigFile load, convert hex values into valid param yaml format, +// unmarshal , and apply beacon chain config file. +func LoadChainConfigFile(chainConfigFileName string) { + yamlFile, err := ioutil.ReadFile(chainConfigFileName) + if err != nil { + log.WithError(err).Fatal("Failed to read chain config file.") + } + // Convert 0x hex inputs to fixed bytes arrays + lines := strings.Split(string(yamlFile), "\n") + for i, line := range lines { + if !strings.HasPrefix(line, "#") && strings.Contains(line, "0x") { + parts := replaceHexStringWithYAMLFormat(line) + lines[i] = strings.Join(parts, "\n") + } + } + yamlFile = []byte(strings.Join(lines, "\n")) + conf := BeaconConfig() + if err := yaml.Unmarshal(yamlFile, conf); err != nil { + log.WithError(err).Fatal("Failed to parse chain config yaml file.") + } + log.Debugf("Config file values: %+v", conf) + OverrideBeaconConfig(conf) +} + +func replaceHexStringWithYAMLFormat(line string) []string { + parts := strings.Split(line, "0x") + b, err := hex.DecodeString(parts[1]) + if err != nil { + log.WithError(err).Error("Failed to decode hex string.") + } + switch l := len(b); { + case l == 1: + var byte byte + byte = b[0] + fixedByte, err := yaml.Marshal(byte) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[0] = parts[0] + string(fixedByte) + parts = parts[:1] + case l > 1 && l <= 4: + var arr [4]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 4 && l <= 8: + var arr [8]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 8 && l <= 16: + var arr [16]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 16 && l <= 20: + var arr [20]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 20 && l <= 32: + var arr [32]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 32 && l <= 48: + var arr [48]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 48 && l <= 64: + var arr [64]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + case l > 64 && l <= 96: + var arr [96]byte + copy(arr[:], b) + fixedByte, err := yaml.Marshal(arr) + if err != nil { + log.WithError(err).Error("Failed to marshal config file.") + } + parts[1] = string(fixedByte) + } + return parts +} diff --git a/shared/params/loader_test.go b/shared/params/loader_test.go new file mode 100644 index 000000000000..aaea940e0912 --- /dev/null +++ b/shared/params/loader_test.go @@ -0,0 +1,117 @@ +package params + +import ( + "path" + "strings" + "testing" + + "github.com/bazelbuild/rules_go/go/tools/bazel" +) + +func TestLoadConfigFile(t *testing.T) { + mainnetConfigFile := ConfigFilePath(t, "mainnet") + LoadChainConfigFile(mainnetConfigFile) + if BeaconConfig().MaxCommitteesPerSlot != MainnetConfig().MaxCommitteesPerSlot { + t.Errorf("Expected MaxCommitteesPerSlot to be set to mainnet value: %d found: %d", + MainnetConfig().MaxCommitteesPerSlot, + BeaconConfig().MaxCommitteesPerSlot) + } + if BeaconConfig().SecondsPerSlot != MainnetConfig().SecondsPerSlot { + t.Errorf("Expected SecondsPerSlot to be set to mainnet value: %d found: %d", + MainnetConfig().SecondsPerSlot, + BeaconConfig().SecondsPerSlot) + } + minimalConfigFile := ConfigFilePath(t, "minimal") + LoadChainConfigFile(minimalConfigFile) + if BeaconConfig().MaxCommitteesPerSlot != MinimalSpecConfig().MaxCommitteesPerSlot { + t.Errorf("Expected MaxCommitteesPerSlot to be set to minimal value: %d found: %d", + MinimalSpecConfig().MaxCommitteesPerSlot, + BeaconConfig().MaxCommitteesPerSlot) + } + if BeaconConfig().SecondsPerSlot != MinimalSpecConfig().SecondsPerSlot { + t.Errorf("Expected SecondsPerSlot to be set to minimal value: %d found: %d", + MinimalSpecConfig().SecondsPerSlot, + BeaconConfig().SecondsPerSlot) + } +} + +func Test_replaceHexStringWithYAMLFormat(t *testing.T) { + + testLines := []struct { + line string + wanted string + }{ + { + line: "ONE_BYTE: 0x41", + wanted: "ONE_BYTE: 65\n", + }, + { + line: "FOUR_BYTES: 0x41414141", + wanted: "FOUR_BYTES: \n- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "THREE_BYTES: 0x414141", + wanted: "THREE_BYTES: \n- 65\n- 65\n- 65\n- 0\n", + }, + { + line: "EIGHT_BYTES: 0x4141414141414141", + wanted: "EIGHT_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "SIXTEEN_BYTES: 0x41414141414141414141414141414141", + wanted: "SIXTEEN_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "TWENTY_BYTES: 0x4141414141414141414141414141414141414141", + wanted: "TWENTY_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "THIRTY_TWO_BYTES: 0x4141414141414141414141414141414141414141414141414141414141414141", + wanted: "THIRTY_TWO_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "FORTY_EIGHT_BYTES: 0x41414141414141414141414141414141414141414141414141414141414141414141" + + "4141414141414141414141414141", + wanted: "FORTY_EIGHT_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n", + }, + { + line: "NINETY_SIX_BYTES: 0x414141414141414141414141414141414141414141414141414141414141414141414141" + + "4141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141" + + "41414141414141414141414141", + wanted: "NINETY_SIX_BYTES: \n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n" + + "- 65\n- 65\n- 65\n- 65\n- 65\n- 65\n", + }, + } + for _, line := range testLines { + parts := replaceHexStringWithYAMLFormat(line.line) + res := strings.Join(parts, "\n") + + if res != line.wanted { + t.Errorf("expected conversion to be: %v got: %v", line.wanted, res) + } + } +} + +// ConfigFilePath sets the proper config and returns the relevant +// config file path from eth2-spec-tests directory. +func ConfigFilePath(t *testing.T, config string) string { + configFolderPath := path.Join("tests", config) + filepath, err := bazel.Runfile(configFolderPath) + if err != nil { + t.Fatal(err) + } + configFilePath := path.Join(filepath, "config.yaml") + return configFilePath +} diff --git a/shared/testutil/spectest.go b/shared/testutil/spectest.go index 8312e1580445..115706e3c4cd 100644 --- a/shared/testutil/spectest.go +++ b/shared/testutil/spectest.go @@ -56,7 +56,6 @@ func TestFolders(t *testing.T, config string, folderPath string) ([]os.FileInfo, return testFolders, testsFolderPath } - // BazelDirectoryNonEmpty returns true if directory exists and is not empty. func BazelDirectoryNonEmpty(filePath string) (bool, error) { p, err := bazel.Runfile(filePath)