Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor RPC to Fully Utilize Ethereum APIs #4243

Merged
merged 30 commits into from
Dec 11, 2019
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
4bdb03a
Merge branch 'v0.9.2' of github.com:prysmaticlabs/prysm into v0.9.2
rauljordan Dec 7, 2019
b87e9a4
include attester as a file in the validator server
rauljordan Dec 7, 2019
db691c0
remove old proposer server impl
rauljordan Dec 7, 2019
635a50c
Merge branch 'v0.9.2' of github.com:prysmaticlabs/prysm into v0.9.2
rauljordan Dec 10, 2019
303b8fc
sync changes
rauljordan Dec 10, 2019
3ca03e5
include new patch and properly sync changes
rauljordan Dec 10, 2019
0a027ea
align with public pbs
rauljordan Dec 10, 2019
6029165
ensure matches rpc def
rauljordan Dec 10, 2019
cf2aa25
fix up status tests
rauljordan Dec 10, 2019
7a9ee57
resolve all broken test files in the validator rpc package
rauljordan Dec 10, 2019
84569fc
gazelle include
rauljordan Dec 10, 2019
d4addeb
fix up the duties implementation
rauljordan Dec 10, 2019
e1faae9
fixed up all get duties functions
rauljordan Dec 10, 2019
3a8773e
all tests pass
rauljordan Dec 10, 2019
7676da9
utilize new ethereum apis
rauljordan Dec 10, 2019
e9739b5
amend validator client to use the new beacon node validator rpc client
rauljordan Dec 10, 2019
5445617
fix up most of validator items
rauljordan Dec 10, 2019
2b864fd
added in mock
rauljordan Dec 10, 2019
5edc4f2
fix up test
rauljordan Dec 10, 2019
92e8673
readd test
rauljordan Dec 10, 2019
f1899ec
add chain serv mock
rauljordan Dec 10, 2019
3605a1a
fix a few more validator methods
rauljordan Dec 11, 2019
8ef3910
all validator tests passingggg
rauljordan Dec 11, 2019
d66b3d0
fix broken test
rauljordan Dec 11, 2019
1af1890
resolve even more broken tests
rauljordan Dec 11, 2019
511a357
all tests passsssss
rauljordan Dec 11, 2019
1d79aaf
fix lint
rauljordan Dec 11, 2019
d48eebe
try PR
rauljordan Dec 11, 2019
3f69c78
fix up test
rauljordan Dec 11, 2019
64c0328
resolve broken other tests
rauljordan Dec 11, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -1249,7 +1249,7 @@ go_repository(

go_repository(
name = "com_github_prysmaticlabs_ethereumapis",
commit = "89a221aac82a7ea3de23d9b8fcd394764a3f3aeb",
commit = "75e46ab40e0a88c3d8238e8746a95ef20d726c34",
importpath = "github.com/prysmaticlabs/ethereumapis",
patch_args = ["-p1"],
patches = [
Expand Down
2 changes: 0 additions & 2 deletions beacon-chain/cache/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ go_library(
visibility = ["//beacon-chain:__subpackages__"],
deps = [
"//proto/beacon/p2p/v1:go_default_library",
"//proto/beacon/rpc/v1:go_default_library",
"//shared/featureconfig:go_default_library",
"//shared/hashutil:go_default_library",
"//shared/params:go_default_library",
Expand Down Expand Up @@ -45,7 +44,6 @@ go_test(
race = "on",
deps = [
"//proto/beacon/p2p/v1:go_default_library",
"//proto/beacon/rpc/v1:go_default_library",
"//shared/featureconfig:go_default_library",
"//shared/hashutil:go_default_library",
"//shared/params:go_default_library",
Expand Down
13 changes: 6 additions & 7 deletions beacon-chain/cache/attestation_data.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to review, just changing argument types

"github.com/prometheus/client_golang/prometheus/promauto"
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
"github.com/prysmaticlabs/prysm/shared/featureconfig"
"k8s.io/client-go/tools/cache"
)
Expand Down Expand Up @@ -59,7 +58,7 @@ func NewAttestationCache() *AttestationCache {

// Get waits for any in progress calculation to complete before returning a
// cached response, if any.
func (c *AttestationCache) Get(ctx context.Context, req *pb.AttestationRequest) (*ethpb.AttestationData, error) {
func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRequest) (*ethpb.AttestationData, error) {
if !featureconfig.Get().EnableAttestationCache {
// Return a miss result if cache is not enabled.
attestationCacheMiss.Inc()
Expand Down Expand Up @@ -113,7 +112,7 @@ func (c *AttestationCache) Get(ctx context.Context, req *pb.AttestationRequest)

// MarkInProgress a request so that any other similar requests will block on
// Get until MarkNotInProgress is called.
func (c *AttestationCache) MarkInProgress(req *pb.AttestationRequest) error {
func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) error {
if !featureconfig.Get().EnableAttestationCache {
return nil
}
Expand All @@ -135,7 +134,7 @@ func (c *AttestationCache) MarkInProgress(req *pb.AttestationRequest) error {

// MarkNotInProgress will release the lock on a given request. This should be
// called after put.
func (c *AttestationCache) MarkNotInProgress(req *pb.AttestationRequest) error {
func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest) error {
if !featureconfig.Get().EnableAttestationCache {
return nil
}
Expand All @@ -151,7 +150,7 @@ func (c *AttestationCache) MarkNotInProgress(req *pb.AttestationRequest) error {
}

// Put the response in the cache.
func (c *AttestationCache) Put(ctx context.Context, req *pb.AttestationRequest, res *ethpb.AttestationData) error {
func (c *AttestationCache) Put(ctx context.Context, req *ethpb.AttestationDataRequest, res *ethpb.AttestationData) error {
if !featureconfig.Get().EnableAttestationCache {
return nil
}
Expand Down Expand Up @@ -180,11 +179,11 @@ func wrapperToKey(i interface{}) (string, error) {
return reqToKey(w.req)
}

func reqToKey(req *pb.AttestationRequest) (string, error) {
func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
return fmt.Sprintf("%d-%d", req.CommitteeIndex, req.Slot), nil
}

type attestationReqResWrapper struct {
req *pb.AttestationRequest
req *ethpb.AttestationDataRequest
res *ethpb.AttestationData
}
3 changes: 1 addition & 2 deletions beacon-chain/cache/attestation_data_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ import (
"github.com/gogo/protobuf/proto"
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
"github.com/prysmaticlabs/prysm/beacon-chain/cache"
pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
)

func TestAttestationCache_RoundTrip(t *testing.T) {
ctx := context.Background()
c := cache.NewAttestationCache()

req := &pb.AttestationRequest{
req := &ethpb.AttestationDataRequest{
CommitteeIndex: 0,
Slot: 1,
}
Expand Down
2 changes: 0 additions & 2 deletions beacon-chain/rpc/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@ go_library(
"//beacon-chain/p2p:go_default_library",
"//beacon-chain/powchain:go_default_library",
"//beacon-chain/rpc/aggregator:go_default_library",
"//beacon-chain/rpc/attester:go_default_library",
"//beacon-chain/rpc/beacon:go_default_library",
"//beacon-chain/rpc/node:go_default_library",
"//beacon-chain/rpc/proposer:go_default_library",
"//beacon-chain/rpc/validator:go_default_library",
"//beacon-chain/sync:go_default_library",
"//proto/beacon/p2p/v1:go_default_library",
Expand Down
48 changes: 0 additions & 48 deletions beacon-chain/rpc/attester/BUILD.bazel

This file was deleted.

1 change: 1 addition & 0 deletions beacon-chain/rpc/beacon/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ go_library(
"//beacon-chain/core/feed:go_default_library",
"//beacon-chain/core/feed/state:go_default_library",
"//beacon-chain/core/helpers:go_default_library",
"//beacon-chain/core/state:go_default_library",
"//beacon-chain/core/validators:go_default_library",
"//beacon-chain/db:go_default_library",
"//beacon-chain/db/filters:go_default_library",
Expand Down
21 changes: 19 additions & 2 deletions beacon-chain/rpc/beacon/blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,15 +175,32 @@ func (bs *Server) chainHeadRetrieval(ctx context.Context) (*ethpb.ChainHead, err
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not get head block root: %v", err)
}

finalizedCheckpoint := headState.FinalizedCheckpoint
justifiedCheckpoint := headState.CurrentJustifiedCheckpoint

if headState.Slot == 0 {
return &ethpb.ChainHead{
HeadSlot: 0,
HeadEpoch: 0,
HeadBlockRoot: headBlockRoot[:],
FinalizedSlot: 0,
FinalizedEpoch: 0,
FinalizedBlockRoot: finalizedCheckpoint.Root,
JustifiedSlot: 0,
JustifiedEpoch: 0,
JustifiedBlockRoot: justifiedCheckpoint.Root,
PreviousJustifiedSlot: 0,
PreviousJustifiedEpoch: 0,
PreviousJustifiedBlockRoot: justifiedCheckpoint.Root,
}, nil
}

b, err := bs.BeaconDB.Block(ctx, bytesutil.ToBytes32(finalizedCheckpoint.Root))
if err != nil || b == nil {
return nil, status.Error(codes.Internal, "Could not get finalized block")
}
finalizedSlot := b.Slot

justifiedCheckpoint := headState.CurrentJustifiedCheckpoint
b, err = bs.BeaconDB.Block(ctx, bytesutil.ToBytes32(justifiedCheckpoint.Root))
if err != nil || b == nil {
return nil, status.Error(codes.Internal, "Could not get justified block")
Expand Down
81 changes: 69 additions & 12 deletions beacon-chain/rpc/beacon/validators.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
"github.com/prysmaticlabs/prysm/shared/bytesutil"
"github.com/prysmaticlabs/prysm/shared/pagination"
Expand Down Expand Up @@ -187,18 +188,24 @@ func (bs *Server) ListValidators(
requestedEpoch = q.Epoch
}

vals := headState.Validators
validatorList := make([]*ethpb.Validators_ValidatorContainer, 0)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Implementing: prysmaticlabs/ethereumapis#75 (including validator indices in listvalidators response)

for i := 0; i < len(headState.Validators); i++ {
validatorList = append(validatorList, &ethpb.Validators_ValidatorContainer{
Index: uint64(i),
Validator: headState.Validators[i],
})
}
if requestedEpoch < currentEpoch {
stopIdx := len(vals)
for idx, val := range vals {
stopIdx := len(validatorList)
for idx, item := range validatorList {
// The first time we see a validator with an activation epoch > the requested epoch,
// we know this validator is from the future relative to what the request wants.
if val.ActivationEpoch > requestedEpoch {
if item.Validator.ActivationEpoch > requestedEpoch {
stopIdx = idx
break
}
}
vals = vals[:stopIdx]
validatorList = validatorList[:stopIdx]
} else if requestedEpoch > currentEpoch {
// Otherwise, we are requesting data from the future and we return an error.
return nil, status.Errorf(
Expand All @@ -210,12 +217,12 @@ func (bs *Server) ListValidators(
}

// Filter active validators if the request specifies it.
res := vals
res := validatorList
if req.Active {
filteredValidators := make([]*ethpb.Validator, 0)
for _, val := range vals {
if helpers.IsActiveValidator(val, requestedEpoch) {
filteredValidators = append(filteredValidators, val)
filteredValidators := make([]*ethpb.Validators_ValidatorContainer, 0)
for _, item := range validatorList {
if helpers.IsActiveValidator(item.Validator, requestedEpoch) {
filteredValidators = append(filteredValidators, item)
}
}
res = filteredValidators
Expand All @@ -226,7 +233,7 @@ func (bs *Server) ListValidators(
// Otherwise, attempting to paginate 0 validators below would result in an error.
if validatorCount == 0 {
return &ethpb.Validators{
Validators: make([]*ethpb.Validator, 0),
ValidatorList: make([]*ethpb.Validators_ValidatorContainer, 0),
TotalSize: int32(0),
NextPageToken: strconv.Itoa(0),
}, nil
Expand All @@ -242,7 +249,7 @@ func (bs *Server) ListValidators(
}

return &ethpb.Validators{
Validators: res[start:end],
ValidatorList: res[start:end],
TotalSize: int32(validatorCount),
NextPageToken: nextPageToken,
}, nil
Expand Down Expand Up @@ -537,3 +544,53 @@ func (bs *Server) GetValidatorQueue(
ExitPublicKeys: exitQueueKeys,
}, nil
}

// GetValidatorPerformance reports the validator's latest balance along with other important metrics on
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moves this method to the beacon RPC as it makes more sense to have it there

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No new logic

// rewards and penalties throughout its lifecycle in the beacon chain.
func (bs *Server) GetValidatorPerformance(
ctx context.Context, req *ethpb.ValidatorPerformanceRequest,
) (*ethpb.ValidatorPerformanceResponse, error) {
headState, err := bs.HeadFetcher.HeadState(ctx)
if err != nil {
return nil, status.Error(codes.Internal, "Could not get head state")
}

// Advance state with empty transitions up to the requested epoch start slot.
if req.Slot > headState.Slot {
headState, err = state.ProcessSlots(ctx, headState, req.Slot)
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not process slots up to %d: %v", req.Slot, err)
}
}

balances := make([]uint64, len(req.PublicKeys))
missingValidators := make([][]byte, 0)
for i, key := range req.PublicKeys {
index, ok, err := bs.BeaconDB.ValidatorIndex(ctx, bytesutil.ToBytes48(key))
if err != nil || !ok {
missingValidators = append(missingValidators, key)
balances[i] = 0
continue
}
balances[i] = headState.Balances[index]
}

activeCount, err := helpers.ActiveValidatorCount(headState, helpers.SlotToEpoch(req.Slot))
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not retrieve active validator count: %v", err)
}

totalActiveBalance, err := helpers.TotalActiveBalance(headState)
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not retrieve total active balance: %v", err)
}

avgBalance := float32(totalActiveBalance / activeCount)
return &ethpb.ValidatorPerformanceResponse{
Balances: balances,
AverageActiveValidatorBalance: avgBalance,
MissingValidators: missingValidators,
TotalValidators: uint64(len(headState.Validators)),
TotalActiveValidators: uint64(activeCount),
}, nil
}
Loading