Decoupling subnet groups

data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/MetaData.json

@@ -21,9 +21,9 @@
       "description" : "Bitvector representing the node's persistent sync committee subnet subscriptions.",
       "format" : "bytes"
     },
-    "custody_subnet_count" : {
+    "custody_group_count" : {
       "type" : "string",
-      "description" : "PeerDAS custody subnet count.",
+      "description" : "PeerDAS custody group count.",
       "example" : "1",
       "format" : "uint64"
     }

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetIdentity.java

@@ -140,9 +140,9 @@ private static SerializableTypeDefinition<MetadataMessage> createMetadataType(
                     "Bitvector representing the node's persistent sync committee subnet subscriptions."),
             MetadataMessage::getOptionalSyncnets)
         .withOptionalField(
-            "custody_subnet_count",
-            UINT64_TYPE.withDescription("PeerDAS custody subnet count."),
-            MetadataMessage::getOptionalCustodySubnetCount)
+            "custody_group_count",
+            UINT64_TYPE.withDescription("PeerDAS custody group count."),
+            MetadataMessage::getOptionalCustodyGroupCount)
         .build();
   }
 

data/serializer/src/main/java/tech/pegasys/teku/api/schema/Metadata.java

@@ -57,23 +57,23 @@ public class Metadata {
   @JsonInclude(JsonInclude.Include.NON_NULL)
   public final String syncCommitteeSubscriptions;
 
-  @JsonProperty("custody_subnet_count")
+  @JsonProperty("custody_group_count")
   @Schema(
       type = "string",
       pattern = PATTERN_UINT8,
-      description = "Uint8 value representing the node's custody subnet count")
-  public final String custodySubnetCount;
+      description = "Uint8 value representing the node's custody group count")
+  public final String custodyGroupCount;
 
   @JsonCreator
   public Metadata(
       @JsonProperty("seq_number") final String sequenceNumber,
       @JsonProperty("attnets") final String attestationSubnetSubscriptions,
       @JsonProperty("syncnets") final String syncCommitteeSubscriptions,
-      @JsonProperty("custody_subnet_count") final String custodySubnetCount) {
+      @JsonProperty("custody_group_count") final String custodyGroupCount) {
     this.sequenceNumber = sequenceNumber;
     this.attestationSubnetSubscriptions = attestationSubnetSubscriptions;
     this.syncCommitteeSubscriptions = syncCommitteeSubscriptions;
-    this.custodySubnetCount = custodySubnetCount;
+    this.custodyGroupCount = custodyGroupCount;
   }
 
   public Metadata(final MetadataMessage metadataMessage) {
@@ -88,19 +88,19 @@ public Metadata(final MetadataMessage metadataMessage) {
               .sszSerialize()
               .toHexString()
               .toLowerCase(Locale.ROOT);
-      this.custodySubnetCount =
-          ((MetadataMessageFulu) metadataMessage).getCustodySubnetCount().toString();
+      this.custodyGroupCount =
+          ((MetadataMessageFulu) metadataMessage).getCustodyGroupCount().toString();
     } else if (metadataMessage instanceof MetadataMessageAltair) {
       this.syncCommitteeSubscriptions =
           ((MetadataMessageAltair) metadataMessage)
               .getSyncnets()
               .sszSerialize()
               .toHexString()
               .toLowerCase(Locale.ROOT);
-      this.custodySubnetCount = null;
+      this.custodyGroupCount = null;
     } else {
       this.syncCommitteeSubscriptions = null;
-      this.custodySubnetCount = null;
+      this.custodyGroupCount = null;
     }
   }
 
@@ -116,7 +116,7 @@ public boolean equals(final Object o) {
     return Objects.equals(sequenceNumber, metadata.sequenceNumber)
         && Objects.equals(attestationSubnetSubscriptions, metadata.attestationSubnetSubscriptions)
         && Objects.equals(syncCommitteeSubscriptions, metadata.syncCommitteeSubscriptions)
-        && Objects.equals(custodySubnetCount, metadata.custodySubnetCount);
+        && Objects.equals(custodyGroupCount, metadata.custodyGroupCount);
   }
 
   @Override
@@ -125,6 +125,6 @@ public int hashCode() {
         sequenceNumber,
         attestationSubnetSubscriptions,
         syncCommitteeSubscriptions,
-        custodySubnetCount);
+        custodyGroupCount);
   }
 }

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/fulu/networking/ComputeColumnsForCustodyGroupTestExecutor.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.reference.fulu.networking;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static tech.pegasys.teku.reference.TestDataUtils.loadYaml;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import tech.pegasys.teku.ethtests.finder.TestDefinition;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.reference.TestExecutor;
+import tech.pegasys.teku.spec.SpecVersion;
+import tech.pegasys.teku.spec.logic.versions.fulu.helpers.MiscHelpersFulu;
+
+public class ComputeColumnsForCustodyGroupTestExecutor implements TestExecutor {
+
+  @Override
+  public void runTest(final TestDefinition testDefinition) throws Exception {
+    final ComputeColumnForCustodyGroupMetaData metaData =
+        loadYaml(testDefinition, "meta.yaml", ComputeColumnForCustodyGroupMetaData.class);
+    final SpecVersion spec = testDefinition.getSpec().getGenesisSpec();
+    final List<UInt64> actualResult =
+        MiscHelpersFulu.required(spec.miscHelpers())
+            .computeColumnsForCustodyGroup(UInt64.valueOf(metaData.getCustodyGroup()));
+    assertThat(new HashSet<>(actualResult)).isEqualTo(metaData.getResult());
+  }
+
+  private static class ComputeColumnForCustodyGroupMetaData {
+
+    @JsonProperty(value = "custody_group", required = true)
+    private int custodyGroup;
+
+    @JsonProperty(value = "result", required = true)
+    private List<Integer> result;
+
+    public int getCustodyGroup() {
+      return custodyGroup;
+    }
+
+    public Set<UInt64> getResult() {
+      return result.stream().map(UInt64::valueOf).collect(Collectors.toUnmodifiableSet());
+    }
+  }
+}

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/fulu/networking/GetCustodyColumnsTestExecutor.java → eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/fulu/networking/GetCustodyGroupsTestExecutor.java

@@ -29,25 +29,25 @@
 import tech.pegasys.teku.spec.SpecVersion;
 import tech.pegasys.teku.spec.logic.versions.fulu.helpers.MiscHelpersFulu;
 
-public class GetCustodyColumnsTestExecutor implements TestExecutor {
+public class GetCustodyGroupsTestExecutor implements TestExecutor {
 
   @Override
   public void runTest(final TestDefinition testDefinition) throws Exception {
-    final GetCustodyColumnsMetaData metaData =
-        loadYaml(testDefinition, "meta.yaml", GetCustodyColumnsMetaData.class);
+    final GetCustodyGroupsMetaData metaData =
+        loadYaml(testDefinition, "meta.yaml", GetCustodyGroupsMetaData.class);
     final SpecVersion spec = testDefinition.getSpec().getGenesisSpec();
     final List<UInt64> actualResult =
         MiscHelpersFulu.required(spec.miscHelpers())
-            .computeCustodyColumnIndexes(metaData.getNodeId(), metaData.getCustodySubnetCount());
+            .getCustodyGroups(metaData.getNodeId(), metaData.getCustodySubnetCount());
     assertThat(new HashSet<>(actualResult)).isEqualTo(metaData.getResult());
   }
 
-  private static class GetCustodyColumnsMetaData {
+  private static class GetCustodyGroupsMetaData {
 
     @JsonProperty(value = "node_id", required = true)
     private String nodeId;
 
-    @JsonProperty(value = "custody_subnet_count", required = true)
+    @JsonProperty(value = "custody_group_count", required = true)
     private int custodySubnetCount;
 
     @JsonProperty(value = "result", required = true)

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/fulu/networking/NetworkingTests.java

@@ -19,6 +19,9 @@
 public class NetworkingTests {
   public static final ImmutableMap<String, TestExecutor> NETWORKING_TEST_TYPES =
       ImmutableMap.<String, TestExecutor>builder()
-          .put("networking/get_custody_columns", new GetCustodyColumnsTestExecutor())
+          .put("networking/get_custody_groups", new GetCustodyGroupsTestExecutor())
+          .put(
+              "networking/compute_columns_for_custody_group",
+              new ComputeColumnsForCustodyGroupTestExecutor())
           .build();
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -1022,12 +1022,6 @@ public boolean isAvailabilityOfDataColumnSidecarsRequiredAtEpoch(
         .isLessThanOrEqualTo(specConfigFulu.getMinEpochsForDataColumnSidecarsRequests());
   }
 
-  public UInt64 computeSubnetForDataColumnSidecar(final DataColumnSidecar dataColumnSidecar) {
-    final SpecConfig config = atSlot(dataColumnSidecar.getSlot()).getConfig();
-    final SpecConfigFulu specConfigFulu = SpecConfigFulu.required(config);
-    return dataColumnSidecar.getIndex().mod(specConfigFulu.getDataColumnSidecarSubnetCount());
-  }
-
   public Optional<UInt64> computeFirstSlotWithBlobSupport() {
     return getSpecConfigDeneb()
         .map(SpecConfigDeneb::getDenebForkEpoch)

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/DelegatingSpecConfigFulu.java

@@ -62,6 +62,11 @@ public int getNumberOfColumns() {
     return delegate.getNumberOfColumns();
   }
 
+  @Override
+  public int getNumberOfCustodyGroups() {
+    return delegate.getNumberOfCustodyGroups();
+  }
+
   @Override
   public int getDataColumnSidecarSubnetCount() {
     return delegate.getDataColumnSidecarSubnetCount();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigFulu.java

@@ -41,6 +41,8 @@ static SpecConfigFulu required(final SpecConfig specConfig) {
 
   int getNumberOfColumns();
 
+  int getNumberOfCustodyGroups();
+
   // networking
   int getDataColumnSidecarSubnetCount();
 

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigFuluImpl.java

@@ -25,6 +25,7 @@ public class SpecConfigFuluImpl extends DelegatingSpecConfigElectra implements S
   private final UInt64 fuluForkEpoch;
 
   private final int numberOfColumns;
+  private final int numberOfCustodyGroups;
   private final int dataColumnSidecarSubnetCount;
   private final int custodyRequirement;
   private final int samplesPerSlot;
@@ -42,6 +43,7 @@ public SpecConfigFuluImpl(
       final UInt64 fieldElementsPerExtBlob,
       final UInt64 kzgCommitmentsInclusionProofDepth,
       final int numberOfColumns,
+      final int numberOfCustodyGroups,
       final int dataColumnSidecarSubnetCount,
       final int custodyRequirement,
       final int samplesPerSlot,
@@ -54,6 +56,7 @@ public SpecConfigFuluImpl(
     this.fieldElementsPerExtBlob = fieldElementsPerExtBlob;
     this.kzgCommitmentsInclusionProofDepth = kzgCommitmentsInclusionProofDepth;
     this.numberOfColumns = numberOfColumns;
+    this.numberOfCustodyGroups = numberOfCustodyGroups;
     this.dataColumnSidecarSubnetCount = dataColumnSidecarSubnetCount;
     this.custodyRequirement = custodyRequirement;
     this.samplesPerSlot = samplesPerSlot;
@@ -91,6 +94,11 @@ public int getNumberOfColumns() {
     return numberOfColumns;
   }
 
+  @Override
+  public int getNumberOfCustodyGroups() {
+    return numberOfCustodyGroups;
+  }
+
   @Override
   public int getDataColumnSidecarSubnetCount() {
     return dataColumnSidecarSubnetCount;
@@ -142,6 +150,7 @@ public boolean equals(final Object o) {
         && Objects.equals(fieldElementsPerExtBlob, that.fieldElementsPerExtBlob)
         && Objects.equals(kzgCommitmentsInclusionProofDepth, that.kzgCommitmentsInclusionProofDepth)
         && numberOfColumns == that.numberOfColumns
+        && numberOfCustodyGroups == that.numberOfCustodyGroups
         && dataColumnSidecarSubnetCount == that.dataColumnSidecarSubnetCount
         && custodyRequirement == that.custodyRequirement
         && minEpochsForDataColumnSidecarsRequests == that.minEpochsForDataColumnSidecarsRequests
@@ -155,6 +164,7 @@ public int hashCode() {
         fuluForkVersion,
         fuluForkEpoch,
         numberOfColumns,
+        numberOfCustodyGroups,
         dataColumnSidecarSubnetCount,
         custodyRequirement,
         fieldElementsPerCell,

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/builder/FuluBuilder.java

@@ -36,6 +36,8 @@ public class FuluBuilder implements ForkConfigBuilder<SpecConfigElectra, SpecCon
   private UInt64 fieldElementsPerExtBlob;
   private UInt64 kzgCommitmentsInclusionProofDepth;
   private Integer numberOfColumns;
+  // FIXME: remove hardcode when Kurtosis config is updated according to the Fulu spec
+  private Integer numberOfCustodyGroups = 128;
   private Integer dataColumnSidecarSubnetCount;
   private Integer custodyRequirement;
   private Integer samplesPerSlot;
@@ -56,6 +58,7 @@ public SpecConfigAndParent<SpecConfigFulu> build(
             fieldElementsPerExtBlob,
             kzgCommitmentsInclusionProofDepth,
             numberOfColumns,
+            numberOfCustodyGroups,
             dataColumnSidecarSubnetCount,
             custodyRequirement,
             samplesPerSlot,
@@ -101,6 +104,12 @@ public FuluBuilder numberOfColumns(final Integer numberOfColumns) {
     return this;
   }
 
+  public FuluBuilder numberOfCustodyGroups(final Integer numberOfCustodyGroups) {
+    checkNotNull(numberOfCustodyGroups);
+    this.numberOfCustodyGroups = numberOfCustodyGroups;
+    return this;
+  }
+
   public FuluBuilder dataColumnSidecarSubnetCount(final Integer dataColumnSidecarSubnetCount) {
     checkNotNull(dataColumnSidecarSubnetCount);
     this.dataColumnSidecarSubnetCount = dataColumnSidecarSubnetCount;
@@ -153,6 +162,7 @@ public Map<String, Object> getValidationMap() {
     constants.put("fuluForkEpoch", fuluForkEpoch);
     constants.put("fuluForkVersion", fuluForkVersion);
     constants.put("numberOfColumns", numberOfColumns);
+    constants.put("numberOfCustodyGroups", numberOfCustodyGroups);
     constants.put("dataColumnSidecarSubnetCount", dataColumnSidecarSubnetCount);
     constants.put("custodyRequirement", custodyRequirement);
     constants.put("samplesPerSlot", samplesPerSlot);

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/networking/libp2p/rpc/metadata/MetadataMessage.java

@@ -33,7 +33,7 @@ default Optional<SszBitvector> getOptionalSyncnets() {
     return Optional.empty();
   }
 
-  default Optional<UInt64> getOptionalCustodySubnetCount() {
+  default Optional<UInt64> getOptionalCustodyGroupCount() {
     return Optional.empty();
   }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/networking/libp2p/rpc/metadata/versions/fulu/MetadataMessageFulu.java

@@ -38,8 +38,8 @@ public class MetadataMessageFulu
       final UInt64 seqNumber,
       final SszBitvector attNets,
       final SszBitvector syncNets,
-      final UInt64 custodySubnetCount) {
-    super(schema, SszUInt64.of(seqNumber), attNets, syncNets, SszUInt64.of(custodySubnetCount));
+      final UInt64 custodyGroupCount) {
+    super(schema, SszUInt64.of(seqNumber), attNets, syncNets, SszUInt64.of(custodyGroupCount));
   }
 
   @Override
@@ -56,7 +56,7 @@ public SszBitvector getSyncnets() {
     return getField2();
   }
 
-  public UInt64 getCustodySubnetCount() {
+  public UInt64 getCustodyGroupCount() {
     return getField3().get();
   }
 
@@ -66,7 +66,7 @@ public Optional<SszBitvector> getOptionalSyncnets() {
   }
 
   @Override
-  public Optional<UInt64> getOptionalCustodySubnetCount() {
-    return Optional.of(getCustodySubnetCount());
+  public Optional<UInt64> getOptionalCustodyGroupCount() {
+    return Optional.of(getCustodyGroupCount());
   }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/networking/libp2p/rpc/metadata/versions/fulu/MetadataMessageSchemaFulu.java

@@ -36,21 +36,21 @@ public MetadataMessageSchemaFulu(final NetworkingSpecConfig networkingSpecConfig
             "attnets", SszBitvectorSchema.create(networkingSpecConfig.getAttestationSubnetCount())),
         namedSchema(
             "syncnets", SszBitvectorSchema.create(NetworkConstants.SYNC_COMMITTEE_SUBNET_COUNT)),
-        namedSchema("custody_subnet_count", SszPrimitiveSchemas.UINT64_SCHEMA));
+        namedSchema("custody_group_count", SszPrimitiveSchemas.UINT64_SCHEMA));
   }
 
   @Override
   public MetadataMessageFulu create(
       final UInt64 seqNumber,
       final Iterable<Integer> attnets,
       final Iterable<Integer> syncnets,
-      final Optional<UInt64> custodySubnetCount) {
+      final Optional<UInt64> custodyGroupCount) {
     return new MetadataMessageFulu(
         this,
         seqNumber,
         getAttnestSchema().ofBits(attnets),
         getSyncnetsSchema().ofBits(syncnets),
-        custodySubnetCount.orElse(UInt64.ZERO));
+        custodyGroupCount.orElse(UInt64.ZERO));
   }
 
   @Override