diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7f276c704c4..9d74043bab2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -3546,7 +3546,7 @@ private String generateConfigurationOverview() { if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) { builder.setTrieLogPruningEnabled(); builder.setTrieLogRetentionThreshold( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold()); + dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad()); builder.setTrieLogPruningLimit( dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit()); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 5b4cf43eb1e..65c13afff38 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -19,7 +19,6 @@ import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; import org.hyperledger.besu.cli.options.CLIOptions; @@ -39,7 +38,8 @@ public class DataStorageOptions implements CLIOptions private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; - private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = + /** The maximum number of historical layers to load. */ + public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; // Use Bonsai DB @@ -54,19 +54,21 @@ public class DataStorageOptions implements CLIOptions names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, paramLabel = "", description = - "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).", + "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using " + + Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED + + " it will also be used as the number of layers of trie logs to retain.", arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); - static class Unstable { + /** The unstable options for data storage. */ + public static class Unstable { private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; - private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = - "--Xbonsai-trie-logs-retention-threshold"; - private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; + /** The bonsai trie log pruning limit. */ + public static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; @CommandLine.Option( hidden = true, @@ -74,13 +76,6 @@ static class Unstable { description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; - @CommandLine.Option( - hidden = true, - names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, - description = - "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") - private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - @CommandLine.Option( hidden = true, names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, @@ -104,19 +99,18 @@ public static DataStorageOptions create() { */ public void validate(final CommandLine commandLine) { if (unstableOptions.bonsaiTrieLogPruningEnabled) { - if (unstableOptions.bonsaiTrieLogRetentionThreshold - < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) { + if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); } if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", + Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT + "=%d must be greater than 0", unstableOptions.bonsaiTrieLogPruningLimit)); } } @@ -128,8 +122,6 @@ static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled = domainObject.getUnstable().getBonsaiTrieLogPruningEnabled(); - dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold = - domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit = domainObject.getUnstable().getBonsaiTrieLogPruningLimit(); @@ -144,7 +136,6 @@ public DataStorageConfiguration toDomainObject() { .unstable( ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled) - .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold) .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit) .build()) .build(); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index c4e924a8354..3433510e18e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; +import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -44,6 +45,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; @@ -56,7 +58,7 @@ public class TrieLogHelper { private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); - static void prune( + void prune( final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, @@ -66,7 +68,7 @@ static void prune( validatePruneConfiguration(config); - final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final long layersToRetain = config.getBonsaiMaxLayersToLoad(); final long chainHeight = blockchain.getChainHeadBlockNumber(); @@ -94,7 +96,7 @@ static void prune( } } - private static void processTrieLogBatches( + private void processTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final long chainHeight, @@ -122,7 +124,7 @@ private static void processTrieLogBatches( } } - private static void saveTrieLogBatches( + private void saveTrieLogBatches( final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogKeys) { @@ -135,7 +137,7 @@ private static void saveTrieLogBatches( } } - private static void restoreTrieLogBatches( + private void restoreTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) { @@ -149,7 +151,7 @@ private static void restoreTrieLogBatches( } } - private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { + private void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { LOG.info("Deleting files..."); @@ -161,7 +163,7 @@ private static void deleteFiles(final String batchFileNameBase, final long numbe } } - private static List getTrieLogKeysForBlocks( + private List getTrieLogKeysForBlocks( final MutableBlockchain blockchain, final long firstBlockOfBatch, final long lastBlockOfBatch) { @@ -175,11 +177,11 @@ private static List getTrieLogKeysForBlocks( return trieLogKeys; } - private static long calculateNumberofBatches(final long layersToRetain) { + private long calculateNumberofBatches(final long layersToRetain) { return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); } - private static boolean validPruneRequirements( + private boolean validPruneRequirements( final MutableBlockchain blockchain, final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor) { @@ -206,7 +208,7 @@ private static boolean validPruneRequirements( return true; } - private static void recreateTrieLogs( + private void recreateTrieLogs( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) @@ -222,7 +224,7 @@ private static void recreateTrieLogs( } } - private static void processTransactionChunk( + private void processTransactionChunk( final int startIndex, final int chunkSize, final List keys, @@ -242,28 +244,32 @@ private static void processTransactionChunk( updater.getTrieLogStorageTransaction().commit(); } - private static void validatePruneConfiguration(final DataStorageConfiguration config) { + @VisibleForTesting + void validatePruneConfiguration(final DataStorageConfiguration config) { checkArgument( - config.getUnstable().getBonsaiTrieLogRetentionThreshold() - >= config.getBonsaiMaxLayersToLoad(), + config.getBonsaiMaxLayersToLoad() + >= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", - config.getBonsaiMaxLayersToLoad())); + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); checkArgument( config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT + + "=%d must be greater than 0", config.getUnstable().getBonsaiTrieLogPruningLimit())); checkArgument( - config.getUnstable().getBonsaiTrieLogPruningLimit() - > config.getUnstable().getBonsaiTrieLogRetentionThreshold(), + config.getUnstable().getBonsaiTrieLogPruningLimit() > config.getBonsaiMaxLayersToLoad(), String.format( - "--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_LIMIT + + "=%d must greater than " + + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + "=%d", config.getUnstable().getBonsaiTrieLogPruningLimit(), - config.getUnstable().getBonsaiTrieLogRetentionThreshold())); + config.getBonsaiMaxLayersToLoad())); } - private static void saveTrieLogsInFile( + private void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) @@ -285,7 +291,7 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { + IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; try (FileInputStream fis = new FileInputStream(batchFileName); @@ -300,7 +306,7 @@ static IdentityHashMap readTrieLogsFromFile(final String batchFi return trieLogs; } - private static void saveTrieLogsAsRlpInFile( + private void saveTrieLogsAsRlpInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) { @@ -325,7 +331,7 @@ private static void saveTrieLogsAsRlpInFile( } } - static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { try { final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); final BytesValueRLPInput input = new BytesValueRLPInput(file, false); @@ -346,7 +352,7 @@ static IdentityHashMap readTrieLogsAsRlpFromFile(final String ba } } - private static IdentityHashMap getTrieLogs( + private IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -359,7 +365,7 @@ private static IdentityHashMap getTrieLogs( return trieLogsToRetain; } - static TrieLogCount getCount( + TrieLogCount getCount( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final int limit, final Blockchain blockchain) { @@ -394,13 +400,13 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - static void printCount(final PrintWriter out, final TrieLogCount count) { + void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", count.total, count.canonicalCount, count.forkCount, count.orphanCount); } - static void importTrieLog( + void importTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); @@ -410,7 +416,7 @@ static void importTrieLog( updater.getTrieLogStorageTransaction().commit(); } - static void exportTrieLog( + void exportTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogHash, final Path directoryPath) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index e624b5f3856..68a97caa73b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -89,14 +89,15 @@ static class CountTrieLog implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final PrintWriter out = spec.commandLine().getOut(); out.println("Counting trie logs..."); - TrieLogHelper.printCount( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.printCount( out, - TrieLogHelper.getCount( + trieLogHelper.getCount( context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); } } @@ -119,11 +120,12 @@ static class PruneTrieLog implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final Path dataDirectoryPath = Paths.get( TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); - TrieLogHelper.prune( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.prune( context.config(), context.rootWorldStateStorage(), context.blockchain(), @@ -173,13 +175,15 @@ public void run() { .toString()); } - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final List listOfBlockHashes = trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + try { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); } catch (IOException e) { throw new RuntimeException(e); @@ -222,8 +226,8 @@ public void run() { } TrieLogContext context = getTrieLogContext(); - - TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); } } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index f2ecbe85c86..8d147f1df9e 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -1078,7 +1078,7 @@ WorldStateArchive createWorldStateArchive( ? new TrieLogPruner( (BonsaiWorldStateKeyValueStorage) worldStateStorage, blockchain, - dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), + dataStorageConfiguration.getBonsaiMaxLayersToLoad(), dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), isProofOfStake) : TrieLogPruner.noOpTrieLogPruner(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 2a63901975b..5b0785174a0 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -42,7 +42,7 @@ public void bonsaiTrieLogPruningLimitOption() { @Test public void bonsaiTrieLogPruningLimitShouldBePositive() { internalTestFailure( - "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", + "--Xbonsai-trie-logs-pruning-limit=0 must be greater than 0", "--Xbonsai-limit-trie-logs-enabled", "--Xbonsai-trie-logs-pruning-limit", "0"); @@ -52,10 +52,10 @@ public void bonsaiTrieLogPruningLimitShouldBePositive() { public void bonsaiTrieLogRetentionThresholdOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "513"); } @@ -63,19 +63,19 @@ public void bonsaiTrieLogRetentionThresholdOption() { public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "512"); } @Test public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { internalTestFailure( - "--Xbonsai-trie-log-retention-threshold minimum value is 512", + "--bonsai-historical-block-limit minimum value is 512", "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "511"); } @@ -88,11 +88,10 @@ protected DataStorageConfiguration createDefaultDomainObject() { protected DataStorageConfiguration createCustomizedDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(100L) + .bonsaiMaxLayersToLoad(513L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiTrieLogPruningEnabled(true) - .bonsaiTrieLogRetentionThreshold(1000L) .bonsaiTrieLogPruningLimit(20) .build()) .build(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 5d4ede7f32e..6a43f53ef56 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -58,6 +58,7 @@ class TrieLogHelperTest { private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; + private TrieLogHelper trieLogHelper; @Mock private MutableBlockchain blockchain; @@ -67,6 +68,11 @@ class TrieLogHelperTest { static BlockHeader blockHeader4; static BlockHeader blockHeader5; + private static class NonValidatingTrieLogHelper extends TrieLogHelper { + @Override + void validatePruneConfiguration(final DataStorageConfiguration config) {} + } + @BeforeEach public void setup() throws IOException { @@ -99,6 +105,8 @@ public void setup() throws IOException { .getTrieLogStorageTransaction() .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); + + trieLogHelper = new NonValidatingTrieLogHelper(); } private static byte[] createTrieLog(final BlockHeader blockHeader) { @@ -122,12 +130,11 @@ public void prune(final @TempDir Path dataDir) throws IOException { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(3L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(3) - .build() - .withBonsaiTrieLogRetentionThreshold(3)) + .bonsaiTrieLogPruningEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -143,7 +150,7 @@ public void prune(final @TempDir Path dataDir) throws IOException { assertArrayEquals( inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); + trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); // assert pruned trie logs are not in the DB assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); @@ -159,16 +166,15 @@ public void prune(final @TempDir Path dataDir) throws IOException { } @Test - public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { + public void cannotPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiTrieLogPruningEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); @@ -177,7 +183,7 @@ public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); } @Test @@ -185,12 +191,11 @@ public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDi DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(10L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(10) - .build() - .withBonsaiTrieLogRetentionThreshold(10)) + .bonsaiTrieLogPruningEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); @@ -198,7 +203,7 @@ public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDi assertThrows( IllegalArgumentException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); } @Test @@ -210,9 +215,8 @@ public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path data .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiTrieLogPruningEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -220,7 +224,7 @@ public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path data assertThrows( IllegalArgumentException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); } @Test @@ -232,15 +236,14 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiTrieLogPruningEnabled(true) + .build()) .build(); assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune( + trieLogHelper.prune( dataStorageConfiguration, inMemoryWorldState, blockchain, @@ -261,13 +264,14 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi @Test public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + trieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() .findFirst() @@ -280,13 +284,14 @@ public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOE @Test public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLogs = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + trieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); @@ -309,13 +314,13 @@ public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); @@ -334,13 +339,13 @@ public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) th new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index a36b150337a..a200aa1f291 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -44,8 +44,7 @@ default Unstable getUnstable() { interface Unstable { boolean DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED = false; - long DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = 512L; - long MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; + long MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; int DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT = 30_000; DataStorageConfiguration.Unstable DEFAULT = @@ -56,11 +55,6 @@ default boolean getBonsaiTrieLogPruningEnabled() { return DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; } - @Value.Default - default long getBonsaiTrieLogRetentionThreshold() { - return DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - } - @Value.Default default int getBonsaiTrieLogPruningLimit() { return DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;