feat: Adding Electra fork support for AggregateAndProof

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Features Added
 - Java 21 for build and runtime. [#995](https://github.com/Consensys/web3signer/pull/995)
-- Electra fork support. [#1020](https://github.com/Consensys/web3signer/pull/1020)
+- Electra fork support. [#1020](https://github.com/Consensys/web3signer/pull/1020) and [#1023](https://github.com/Consensys/web3signer/pull/1023)
 
 ### Bugs fixed
 - Override protobuf-java to 3.25.5 which is a transitive dependency from google-cloud-secretmanager. It fixes CVE-2024-7254.

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/utils/Eth2AggregateAndProofSigningRequestUtil.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.web3signer.dsl.utils;
+
+import tech.pegasys.teku.api.schema.Fork;
+import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.datastructures.operations.AggregateAndProof;
+import tech.pegasys.teku.spec.datastructures.state.ForkInfo;
+import tech.pegasys.teku.spec.signatures.SigningRootUtil;
+import tech.pegasys.teku.spec.util.DataStructureUtil;
+import tech.pegasys.web3signer.core.service.http.ArtifactType;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.AggregateAndProofV2;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.Eth2SigningRequestBody;
+
+import org.apache.tuweni.bytes.Bytes;
+
+public class Eth2AggregateAndProofSigningRequestUtil {
+  private final SpecMilestone specMilestone;
+  private final DataStructureUtil dataStructureUtil;
+  private final SigningRootUtil signingRootUtil;
+  private final ForkInfo tekuForkInfo;
+  private final Fork tekuFork;
+  private final tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.ForkInfo forkInfo;
+  private final Bytes signingRoot;
+  private final AggregateAndProof aggregateAndProof;
+
+  public Eth2AggregateAndProofSigningRequestUtil(final SpecMilestone specMilestone) {
+    final Spec spec = TestSpecFactory.createMinimal(specMilestone);
+    this.specMilestone = specMilestone;
+    dataStructureUtil = new DataStructureUtil(spec);
+    signingRootUtil = new SigningRootUtil(spec);
+    tekuForkInfo = Eth2RequestUtils.forkInfo().asInternalForkInfo();
+    tekuFork = new Fork(tekuForkInfo.getFork());
+    forkInfo =
+        new tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.ForkInfo(
+            tekuFork, tekuForkInfo.getGenesisValidatorsRoot());
+    aggregateAndProof = dataStructureUtil.randomAggregateAndProof();
+    signingRoot =
+        signingRootUtil.signingRootForSignAggregateAndProof(aggregateAndProof, tekuForkInfo);
+  }
+
+  public Eth2SigningRequestBody createAggregateAndProofV2Request() {
+    final tech.pegasys.teku.api.schema.AggregateAndProof aggregateAndProof =
+        new tech.pegasys.teku.api.schema.AggregateAndProof(this.aggregateAndProof);
+    return Eth2SigningRequestBodyBuilder.anEth2SigningRequestBody()
+        .withType(ArtifactType.AGGREGATE_AND_PROOF_V2)
+        .withSigningRoot(signingRoot)
+        .withForkInfo(forkInfo)
+        .withAggregateAndProofV2(new AggregateAndProofV2(specMilestone, aggregateAndProof))
+        .build();
+  }
+}

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/utils/Eth2RequestUtils.java

@@ -40,6 +40,7 @@
 import tech.pegasys.teku.spec.signatures.SigningRootUtil;
 import tech.pegasys.teku.spec.util.DataStructureUtil;
 import tech.pegasys.web3signer.core.service.http.ArtifactType;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.AggregateAndProofV2;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.AggregationSlot;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.DepositMessage;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.Eth2SigningRequestBody;
@@ -104,7 +105,9 @@ public static Eth2SigningRequestBody createCannedRequest(final ArtifactType arti
       case AGGREGATION_SLOT:
         return createAggregationSlot();
       case AGGREGATE_AND_PROOF:
-        return createAggregateAndProof();
+        return createAggregateAndProof(true);
+      case AGGREGATE_AND_PROOF_V2:
+        return createAggregateAndProof(false);
       case SYNC_COMMITTEE_MESSAGE:
         return createSyncCommitteeMessageRequest();
       case SYNC_COMMITTEE_SELECTION_PROOF:
@@ -118,7 +121,7 @@ public static Eth2SigningRequestBody createCannedRequest(final ArtifactType arti
     }
   }
 
-  private static Eth2SigningRequestBody createAggregateAndProof() {
+  private static Eth2SigningRequestBody createAggregateAndProof(final boolean isLegacy) {
     final ForkInfo forkInfo = forkInfo();
     final Bytes sszBytes = Bytes.of(0, 0, 1, 1);
     final AttestationData attestationData =
@@ -136,7 +139,7 @@ private static Eth2SigningRequestBody createAggregateAndProof() {
         new Attestation(
             sszBytes,
             attestationData,
-            sszBytes, // committee_bits
+            null, // committee_bits in pre-Electra Attestation must be null
             BLSSignature.fromHexString(
                 "0xa627242e4a5853708f4ebf923960fb8192f93f2233cd347e05239d86dd9fb66b721ceec1baeae6647f498c9126074f1101a87854d674b6eebc220fd8c3d8405bdfd8e286b707975d9e00a56ec6cbbf762f23607d490f0bbb16c3e0e483d51875"));
     final BLSSignature selectionProof =
@@ -148,10 +151,11 @@ private static Eth2SigningRequestBody createAggregateAndProof() {
         SIGNING_ROOT_UTIL.signingRootForSignAggregateAndProof(
             aggregateAndProof.asInternalAggregateAndProof(SPEC), forkInfo.asInternalForkInfo());
     return Eth2SigningRequestBodyBuilder.anEth2SigningRequestBody()
-        .withType(ArtifactType.AGGREGATE_AND_PROOF)
+        .withType(isLegacy ? ArtifactType.AGGREGATE_AND_PROOF : ArtifactType.AGGREGATE_AND_PROOF_V2)
         .withSigningRoot(signingRoot)
         .withForkInfo(forkInfo)
-        .withAggregateAndProof(aggregateAndProof)
+        .withAggregateAndProofV2(
+            new AggregateAndProofV2(isLegacy ? null : SpecMilestone.PHASE0, aggregateAndProof))
         .build();
   }
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/utils/Eth2SigningRequestBodyBuilder.java

@@ -12,12 +12,12 @@
  */
 package tech.pegasys.web3signer.dsl.utils;
 
-import tech.pegasys.teku.api.schema.AggregateAndProof;
 import tech.pegasys.teku.api.schema.AttestationData;
 import tech.pegasys.teku.api.schema.BeaconBlock;
 import tech.pegasys.teku.api.schema.VoluntaryExit;
 import tech.pegasys.teku.api.schema.altair.ContributionAndProof;
 import tech.pegasys.web3signer.core.service.http.ArtifactType;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.AggregateAndProofV2;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.AggregationSlot;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.BlockRequest;
 import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.DepositMessage;
@@ -38,7 +38,7 @@ public final class Eth2SigningRequestBodyBuilder {
   private BlockRequest blockRequest;
   private AttestationData attestation;
   private AggregationSlot aggregationSlot;
-  private AggregateAndProof aggregateAndProof;
+  private AggregateAndProofV2 aggregateAndProofV2;
   private VoluntaryExit voluntaryExit;
   private RandaoReveal randaoReveal;
   private DepositMessage deposit;
@@ -88,8 +88,9 @@ public Eth2SigningRequestBodyBuilder withAggregationSlot(AggregationSlot aggrega
     return this;
   }
 
-  public Eth2SigningRequestBodyBuilder withAggregateAndProof(AggregateAndProof aggregateAndProof) {
-    this.aggregateAndProof = aggregateAndProof;
+  public Eth2SigningRequestBodyBuilder withAggregateAndProofV2(
+      AggregateAndProofV2 aggregateAndProofV2) {
+    this.aggregateAndProofV2 = aggregateAndProofV2;
     return this;
   }
 
@@ -141,7 +142,7 @@ public Eth2SigningRequestBody build() {
         blockRequest,
         attestation,
         aggregationSlot,
-        aggregateAndProof,
+        aggregateAndProofV2,
         voluntaryExit,
         randaoReveal,
         deposit,

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/BlsSigningAcceptanceTest.java

@@ -63,7 +63,7 @@ public class BlsSigningAcceptanceTest extends SigningAcceptanceTestBase {
   private static final BLSKeyPair KEY_PAIR = new BLSKeyPair(KEY);
   private static final BLSPublicKey PUBLIC_KEY = KEY_PAIR.getPublicKey();
 
-  @ParameterizedTest
+  @ParameterizedTest(name = "{index} - {0} Signing")
   @EnumSource
   public void signDataWithKeyLoadedFromUnencryptedFile(final ArtifactType artifactType)
       throws JsonProcessingException {
@@ -253,7 +253,7 @@ public void failsIfSigningRootDoesNotMatchSigningData(final ArtifactType artifac
             request.blockRequest(),
             request.attestation(),
             request.aggregationSlot(),
-            request.aggregateAndProof(),
+            request.aggregateAndProofV2(),
             request.voluntaryExit(),
             request.randaoReveal(),
             request.deposit(),
@@ -291,7 +291,7 @@ public void ableToSignWithoutSigningRootField(final ContentType acceptableConten
             request.blockRequest(),
             request.attestation(),
             request.aggregationSlot(),
-            request.aggregateAndProof(),
+            request.aggregateAndProofV2(),
             request.voluntaryExit(),
             request.randaoReveal(),
             request.deposit(),

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/Eth2AggregateAndProofSigningAcceptanceTest.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2021 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.web3signer.tests.signing;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import tech.pegasys.teku.api.schema.Attestation;
+import tech.pegasys.teku.bls.BLS;
+import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.teku.bls.BLSPublicKey;
+import tech.pegasys.teku.bls.BLSSecretKey;
+import tech.pegasys.teku.bls.BLSSignature;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.networks.Eth2Network;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.Eth2SigningRequestBody;
+import tech.pegasys.web3signer.dsl.utils.Eth2AggregateAndProofSigningRequestUtil;
+import tech.pegasys.web3signer.dsl.utils.MetadataFileHelpers;
+import tech.pegasys.web3signer.signing.KeyType;
+
+import java.nio.file.Path;
+
+import io.restassured.http.ContentType;
+import io.restassured.response.Response;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+public class Eth2AggregateAndProofSigningAcceptanceTest extends SigningAcceptanceTestBase {
+  private static final String PRIVATE_KEY =
+      "3ee2224386c82ffea477e2adf28a2929f5c349165a4196158c7f3a2ecca40f35";
+
+  private static final MetadataFileHelpers METADATA_FILE_HELPERS = new MetadataFileHelpers();
+  private static final BLSSecretKey KEY =
+      BLSSecretKey.fromBytes(Bytes32.fromHexString(PRIVATE_KEY));
+  private static final BLSKeyPair KEY_PAIR = new BLSKeyPair(KEY);
+  private static final BLSPublicKey PUBLIC_KEY = KEY_PAIR.getPublicKey();
+
+  @BeforeEach
+  void setup() {
+    final String configFilename = PUBLIC_KEY.toString().substring(2);
+    final Path keyConfigFile = testDirectory.resolve(configFilename + ".yaml");
+    METADATA_FILE_HELPERS.createUnencryptedYamlFileAt(keyConfigFile, PRIVATE_KEY, KeyType.BLS);
+  }
+
+  @ParameterizedTest(name = "#{index} - {0}: Sign and verify AggregateAndProofV2 Signature")
+  @EnumSource(
+      value = SpecMilestone.class,
+      names = {"PHASE0", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB", "ELECTRA"})
+  void signAndVerifyAggregateAndProofV2Signature(final SpecMilestone specMilestone)
+      throws Exception {
+    final Eth2AggregateAndProofSigningRequestUtil util =
+        new Eth2AggregateAndProofSigningRequestUtil(specMilestone);
+
+    setupEth2Signer(Eth2Network.MINIMAL, specMilestone);
+
+    final Eth2SigningRequestBody request = util.createAggregateAndProofV2Request();
+    // assert that the Attestation in AggregateAndProofV2 is created correctly for the spec
+    final Attestation aggregate = request.aggregateAndProofV2().data().aggregate;
+    if (specMilestone.isGreaterThanOrEqualTo(SpecMilestone.ELECTRA)) {
+      assertThat(aggregate.committee_bits).isNotNull();
+    } else {
+      assertThat(aggregate.committee_bits).isNull();
+    }
+    final Response response =
+        signer.eth2Sign(KEY_PAIR.getPublicKey().toString(), request, ContentType.JSON);
+    final Bytes signature = verifyAndGetSignatureResponse(response, ContentType.JSON);
+    final BLSSignature expectedSignature = BLS.sign(KEY_PAIR.getSecretKey(), request.signingRoot());
+    assertThat(signature).isEqualTo(expectedSignature.toBytesCompressed());
+  }
+}

core/src/main/java/tech/pegasys/web3signer/core/service/http/ArtifactType.java

@@ -17,7 +17,9 @@ public enum ArtifactType {
   BLOCK_V2,
   ATTESTATION,
   AGGREGATION_SLOT,
+  @Deprecated(since = "1.2.0") // Deprecated in Remoting API Spec v1.2.0
   AGGREGATE_AND_PROOF,
+  AGGREGATE_AND_PROOF_V2,
   DEPOSIT,
   RANDAO_REVEAL,
   VOLUNTARY_EXIT,

core/src/main/java/tech/pegasys/web3signer/core/service/http/handlers/signing/eth2/AggregateAndProofV2.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.web3signer.core.service.http.handlers.signing.eth2;
+
+import tech.pegasys.teku.api.schema.AggregateAndProof;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.json.AggregateAndProofV2Deserializer;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.json.AggregateAndProofV2Serializer;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+/**
+ * Represents an AGGREGATE_AND_PROOF_V2 signing request.
+ *
+ * <p>This class is designed to handle both the new format (with explicit version and data fields)
+ * and the legacy format of AGGREGATE_AND_PROOF signing requests. The aggregate property (i.e.,
+ * Attestation) has introduced a new field 'committee_bits' in the Electra spec.
+ *
+ * <p>The Teku signing utility is able to utilize the same Attestation object for all changes,
+ * ensuring compatibility across different versions.
+ *
+ * <p>Deserialization is handled by {@link AggregateAndProofV2Deserializer}, which supports both:
+ *
+ * <ul>
+ *   <li>New format: JSON with explicit "version" and "data" fields
+ *   <li>Legacy format: JSON without "version" and "data" fields (treated as legacy
+ *       AggregateAndProof)
+ * </ul>
+ *
+ * <p>For legacy format, the deserializer sets the version to null and treats the entire JSON as the
+ * data field.
+ *
+ * <p>Serialization is handled by {@link AggregateAndProofV2Serializer}, which: *
+ *
+ * <ul>
+ *   *
+ *   <li>For new format (version is not null): Serializes with "version" and "data" fields *
+ *   <li>For legacy format (version is null): Serializes all data fields at the top level *
+ *   <li>Throws a JsonMappingException if the data field is null *
+ * </ul>
+ *
+ * *
+ */
+@JsonDeserialize(using = AggregateAndProofV2Deserializer.class)
+@JsonSerialize(using = AggregateAndProofV2Serializer.class)
+public record AggregateAndProofV2(
+    @JsonProperty(value = "version") SpecMilestone version,
+    @JsonProperty(value = "data") AggregateAndProof data) {}

core/src/main/java/tech/pegasys/web3signer/core/service/http/handlers/signing/eth2/Eth2SignForIdentifierHandler.java

@@ -260,9 +260,10 @@ private Bytes computeSigningRoot(final Eth2SigningRequestBody body) {
         return signingRootUtil.signingRootForSignAttestationData(
             body.attestation().asInternalAttestationData(), body.forkInfo().asInternalForkInfo());
       case AGGREGATE_AND_PROOF:
-        checkArgument(body.aggregateAndProof() != null, "aggregateAndProof must be specified");
+      case AGGREGATE_AND_PROOF_V2:
+        checkArgument(body.aggregateAndProofV2() != null, "aggregateAndProof must be specified");
         return signingRootUtil.signingRootForSignAggregateAndProof(
-            body.aggregateAndProof().asInternalAggregateAndProof(eth2Spec),
+            body.aggregateAndProofV2().data().asInternalAggregateAndProof(eth2Spec),
             body.forkInfo().asInternalForkInfo());
       case AGGREGATION_SLOT:
         checkArgument(body.aggregationSlot() != null, "aggregationSlot must be specified");

core/src/main/java/tech/pegasys/web3signer/core/service/http/handlers/signing/eth2/Eth2SigningRequestBody.java

@@ -12,7 +12,6 @@
  */
 package tech.pegasys.web3signer.core.service.http.handlers.signing.eth2;
 
-import tech.pegasys.teku.api.schema.AggregateAndProof;
 import tech.pegasys.teku.api.schema.AttestationData;
 import tech.pegasys.teku.api.schema.BeaconBlock;
 import tech.pegasys.teku.api.schema.VoluntaryExit;
@@ -31,7 +30,7 @@ public record Eth2SigningRequestBody(
     @JsonProperty("beacon_block") BlockRequest blockRequest,
     @JsonProperty("attestation") AttestationData attestation,
     @JsonProperty("aggregation_slot") AggregationSlot aggregationSlot,
-    @JsonProperty("aggregate_and_proof") AggregateAndProof aggregateAndProof,
+    @JsonProperty("aggregate_and_proof") AggregateAndProofV2 aggregateAndProofV2,
     @JsonProperty("voluntary_exit") VoluntaryExit voluntaryExit,
     @JsonProperty("randao_reveal") RandaoReveal randaoReveal,
     @JsonProperty("deposit") DepositMessage deposit,