Reuse --bonsai-historical-block-limit instead of --Xbonsai-trie-log-r…

…etention-threshold

Signed-off-by: Simon Dudley <[email protected]>

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -3546,7 +3546,7 @@ private String generateConfigurationOverview() {
     if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) {
       builder.setTrieLogPruningEnabled();
       builder.setTrieLogRetentionThreshold(
-          dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold());
+          dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad());
       builder.setTrieLogPruningLimit(
           dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit());
     }

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -19,7 +19,6 @@
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
-import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
 
 import org.hyperledger.besu.cli.options.CLIOptions;
@@ -39,7 +38,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
 
   private static final String DATA_STORAGE_FORMAT = "--data-storage-format";
 
-  private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
+  /** The maximum number of historical layers to load. */
+  public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
       "--bonsai-historical-block-limit";
 
   // Use Bonsai DB
@@ -54,33 +54,28 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
       names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
       paramLabel = "<LONG>",
       description =
-          "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).",
+          "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using "
+              + Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED
+              + " it will also be used as the number of layers of trie logs to retain.",
       arity = "1")
   private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 
   @CommandLine.ArgGroup(validate = false)
   private final DataStorageOptions.Unstable unstableOptions = new Unstable();
 
-  static class Unstable {
+  /** The unstable options for data storage. */
+  public static class Unstable {
     private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED =
         "--Xbonsai-limit-trie-logs-enabled";
-    private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD =
-        "--Xbonsai-trie-logs-retention-threshold";
-    private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit";
+    /** The bonsai trie log pruning limit. */
+    public static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit";
 
     @CommandLine.Option(
         hidden = true,
         names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED},
         description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})")
     private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
 
-    @CommandLine.Option(
-        hidden = true,
-        names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD},
-        description =
-            "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})")
-    private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
-
     @CommandLine.Option(
         hidden = true,
         names = {BONSAI_TRIE_LOG_PRUNING_LIMIT},
@@ -104,19 +99,18 @@ public static DataStorageOptions create() {
    */
   public void validate(final CommandLine commandLine) {
     if (unstableOptions.bonsaiTrieLogPruningEnabled) {
-      if (unstableOptions.bonsaiTrieLogRetentionThreshold
-          < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) {
+      if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) {
         throw new CommandLine.ParameterException(
             commandLine,
             String.format(
-                "--Xbonsai-trie-log-retention-threshold minimum value is %d",
+                BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
                 MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD));
       }
       if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) {
         throw new CommandLine.ParameterException(
             commandLine,
             String.format(
-                "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
+                Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT + "=%d must be greater than 0",
                 unstableOptions.bonsaiTrieLogPruningLimit));
       }
     }
@@ -128,8 +122,6 @@ static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject
     dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
     dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled =
         domainObject.getUnstable().getBonsaiTrieLogPruningEnabled();
-    dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold =
-        domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold();
     dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit =
         domainObject.getUnstable().getBonsaiTrieLogPruningLimit();
 
@@ -144,7 +136,6 @@ public DataStorageConfiguration toDomainObject() {
         .unstable(
             ImmutableDataStorageConfiguration.Unstable.builder()
                 .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled)
-                .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold)
                 .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit)
                 .build())
         .build();

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java

@@ -18,6 +18,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
 
+import org.hyperledger.besu.cli.options.stable.DataStorageOptions;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
@@ -44,6 +45,7 @@
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.slf4j.Logger;
@@ -56,7 +58,7 @@ public class TrieLogHelper {
   private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000;
   private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class);
 
-  static void prune(
+  void prune(
       final DataStorageConfiguration config,
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final MutableBlockchain blockchain,
@@ -66,7 +68,7 @@ static void prune(
 
     validatePruneConfiguration(config);
 
-    final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold();
+    final long layersToRetain = config.getBonsaiMaxLayersToLoad();
 
     final long chainHeight = blockchain.getChainHeadBlockNumber();
 
@@ -94,7 +96,7 @@ static void prune(
     }
   }
 
-  private static void processTrieLogBatches(
+  private void processTrieLogBatches(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final MutableBlockchain blockchain,
       final long chainHeight,
@@ -122,7 +124,7 @@ private static void processTrieLogBatches(
     }
   }
 
-  private static void saveTrieLogBatches(
+  private void saveTrieLogBatches(
       final String batchFileName,
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final List<Hash> trieLogKeys) {
@@ -135,7 +137,7 @@ private static void saveTrieLogBatches(
     }
   }
 
-  private static void restoreTrieLogBatches(
+  private void restoreTrieLogBatches(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final long batchNumber,
       final String batchFileNameBase) {
@@ -149,7 +151,7 @@ private static void restoreTrieLogBatches(
     }
   }
 
-  private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) {
+  private void deleteFiles(final String batchFileNameBase, final long numberOfBatches) {
 
     LOG.info("Deleting files...");
 
@@ -161,7 +163,7 @@ private static void deleteFiles(final String batchFileNameBase, final long numbe
     }
   }
 
-  private static List<Hash> getTrieLogKeysForBlocks(
+  private List<Hash> getTrieLogKeysForBlocks(
       final MutableBlockchain blockchain,
       final long firstBlockOfBatch,
       final long lastBlockOfBatch) {
@@ -175,11 +177,11 @@ private static List<Hash> getTrieLogKeysForBlocks(
     return trieLogKeys;
   }
 
-  private static long calculateNumberofBatches(final long layersToRetain) {
+  private long calculateNumberofBatches(final long layersToRetain) {
     return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1);
   }
 
-  private static boolean validPruneRequirements(
+  private boolean validPruneRequirements(
       final MutableBlockchain blockchain,
       final long chainHeight,
       final long lastBlockNumberToRetainTrieLogsFor) {
@@ -206,7 +208,7 @@ private static boolean validPruneRequirements(
     return true;
   }
 
-  private static void recreateTrieLogs(
+  private void recreateTrieLogs(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final long batchNumber,
       final String batchFileNameBase)
@@ -222,7 +224,7 @@ private static void recreateTrieLogs(
     }
   }
 
-  private static void processTransactionChunk(
+  private void processTransactionChunk(
       final int startIndex,
       final int chunkSize,
       final List<byte[]> keys,
@@ -242,28 +244,32 @@ private static void processTransactionChunk(
     updater.getTrieLogStorageTransaction().commit();
   }
 
-  private static void validatePruneConfiguration(final DataStorageConfiguration config) {
+  @VisibleForTesting
+  void validatePruneConfiguration(final DataStorageConfiguration config) {
     checkArgument(
-        config.getUnstable().getBonsaiTrieLogRetentionThreshold()
-            >= config.getBonsaiMaxLayersToLoad(),
+        config.getBonsaiMaxLayersToLoad()
+            >= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD,
         String.format(
-            "--Xbonsai-trie-log-retention-threshold minimum value is %d",
-            config.getBonsaiMaxLayersToLoad()));
+            DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
+            DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD));
     checkArgument(
         config.getUnstable().getBonsaiTrieLogPruningLimit() > 0,
         String.format(
-            "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
+            DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT
+                + "=%d must be greater than 0",
             config.getUnstable().getBonsaiTrieLogPruningLimit()));
     checkArgument(
-        config.getUnstable().getBonsaiTrieLogPruningLimit()
-            > config.getUnstable().getBonsaiTrieLogRetentionThreshold(),
+        config.getUnstable().getBonsaiTrieLogPruningLimit() > config.getBonsaiMaxLayersToLoad(),
         String.format(
-            "--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d",
+            DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT
+                + "=%d must greater than "
+                + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+                + "=%d",
             config.getUnstable().getBonsaiTrieLogPruningLimit(),
-            config.getUnstable().getBonsaiTrieLogRetentionThreshold()));
+            config.getBonsaiMaxLayersToLoad()));
   }
 
-  private static void saveTrieLogsInFile(
+  private void saveTrieLogsInFile(
       final List<Hash> trieLogsKeys,
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final String batchFileName)
@@ -285,7 +291,7 @@ private static void saveTrieLogsInFile(
   }
 
   @SuppressWarnings("unchecked")
-  static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {
+  IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {
 
     IdentityHashMap<byte[], byte[]> trieLogs;
     try (FileInputStream fis = new FileInputStream(batchFileName);
@@ -300,7 +306,7 @@ static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFi
     return trieLogs;
   }
 
-  private static void saveTrieLogsAsRlpInFile(
+  private void saveTrieLogsAsRlpInFile(
       final List<Hash> trieLogsKeys,
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final String batchFileName) {
@@ -325,7 +331,7 @@ private static void saveTrieLogsAsRlpInFile(
     }
   }
 
-  static IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String batchFileName) {
+  IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String batchFileName) {
     try {
       final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName)));
       final BytesValueRLPInput input = new BytesValueRLPInput(file, false);
@@ -346,7 +352,7 @@ static IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String ba
     }
   }
 
-  private static IdentityHashMap<byte[], byte[]> getTrieLogs(
+  private IdentityHashMap<byte[], byte[]> getTrieLogs(
       final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) {
     IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>();
 
@@ -359,7 +365,7 @@ private static IdentityHashMap<byte[], byte[]> getTrieLogs(
     return trieLogsToRetain;
   }
 
-  static TrieLogCount getCount(
+  TrieLogCount getCount(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final int limit,
       final Blockchain blockchain) {
@@ -394,13 +400,13 @@ static TrieLogCount getCount(
     return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get());
   }
 
-  static void printCount(final PrintWriter out, final TrieLogCount count) {
+  void printCount(final PrintWriter out, final TrieLogCount count) {
     out.printf(
         "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n",
         count.total, count.canonicalCount, count.forkCount, count.orphanCount);
   }
 
-  static void importTrieLog(
+  void importTrieLog(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) {
 
     var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString());
@@ -410,7 +416,7 @@ static void importTrieLog(
     updater.getTrieLogStorageTransaction().commit();
   }
 
-  static void exportTrieLog(
+  void exportTrieLog(
       final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
       final List<Hash> trieLogHash,
       final Path directoryPath)

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -89,14 +89,15 @@ static class CountTrieLog implements Runnable {
 
     @Override
     public void run() {
-      TrieLogContext context = getTrieLogContext();
+      final TrieLogContext context = getTrieLogContext();
 
       final PrintWriter out = spec.commandLine().getOut();
 
       out.println("Counting trie logs...");
-      TrieLogHelper.printCount(
+      final TrieLogHelper trieLogHelper = new TrieLogHelper();
+      trieLogHelper.printCount(
           out,
-          TrieLogHelper.getCount(
+          trieLogHelper.getCount(
               context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain));
     }
   }
@@ -119,11 +120,12 @@ static class PruneTrieLog implements Runnable {
 
     @Override
     public void run() {
-      TrieLogContext context = getTrieLogContext();
+      final TrieLogContext context = getTrieLogContext();
       final Path dataDirectoryPath =
           Paths.get(
               TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString());
-      TrieLogHelper.prune(
+      final TrieLogHelper trieLogHelper = new TrieLogHelper();
+      trieLogHelper.prune(
           context.config(),
           context.rootWorldStateStorage(),
           context.blockchain(),
@@ -173,13 +175,15 @@ public void run() {
                     .toString());
       }
 
-      TrieLogContext context = getTrieLogContext();
+      final TrieLogContext context = getTrieLogContext();
 
       final List<Hash> listOfBlockHashes =
           trieLogBlockHashList.stream().map(Hash::fromHexString).toList();
 
+      final TrieLogHelper trieLogHelper = new TrieLogHelper();
+
       try {
-        TrieLogHelper.exportTrieLog(
+        trieLogHelper.exportTrieLog(
             context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath);
       } catch (IOException e) {
         throw new RuntimeException(e);
@@ -222,8 +226,8 @@ public void run() {
       }
 
       TrieLogContext context = getTrieLogContext();
-
-      TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath);
+      final TrieLogHelper trieLogHelper = new TrieLogHelper();
+      trieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath);
     }
   }