From 9b2807fda1c472c6ca2601c1ac7ec0fa0819e423 Mon Sep 17 00:00:00 2001 From: Davis Zhang Date: Fri, 17 Jan 2025 11:56:01 -0800 Subject: [PATCH] repro --- .../client/TestHoodieClientMultiWriter.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index cf509376e306d..39cfc1ca1d38c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -152,25 +152,6 @@ public void clean() throws IOException { cleanupResources(); } - private static final List LOCK_PROVIDER_CLASSES = Arrays.asList( - InProcessLockProvider.class, - FileSystemBasedLockProvider.class); - - private static final List CONFLICT_RESOLUTION_STRATEGY_CLASSES = Arrays.asList( - new SimpleConcurrentFileWritesConflictResolutionStrategy(), - new PreferWriterConflictResolutionStrategy()); - - private static Iterable providerClassResolutionStrategyAndTableType() { - List opts = new ArrayList<>(); - for (Object providerClass : LOCK_PROVIDER_CLASSES) { - for (ConflictResolutionStrategy resolutionStrategy : CONFLICT_RESOLUTION_STRATEGY_CLASSES) { - opts.add(new Object[] {HoodieTableType.COPY_ON_WRITE, providerClass, resolutionStrategy}); - opts.add(new Object[] {HoodieTableType.MERGE_ON_READ, providerClass, resolutionStrategy}); - } - } - return opts; - } - @ParameterizedTest @MethodSource("configParamsDirectBased") public void testHoodieClientBasicMultiWriterWithEarlyConflictDetectionDirect(String tableType, String earlyConflictDetectionStrategy) throws Exception { @@ -499,6 +480,26 @@ private void latchCountDownAndWait(CountDownLatch latch, long waitTimeMillis) { } } + private static final List LOCK_PROVIDER_CLASSES = Arrays.asList( +// InProcessLockProvider.class, + FileSystemBasedLockProvider.class); + + private static final List CONFLICT_RESOLUTION_STRATEGY_CLASSES = Arrays.asList( + new SimpleConcurrentFileWritesConflictResolutionStrategy() +// new PreferWriterConflictResolutionStrategy() + ); + + private static Iterable providerClassResolutionStrategyAndTableType() { + List opts = new ArrayList<>(); + for (Object providerClass : LOCK_PROVIDER_CLASSES) { + for (ConflictResolutionStrategy resolutionStrategy : CONFLICT_RESOLUTION_STRATEGY_CLASSES) { +// opts.add(new Object[] {HoodieTableType.COPY_ON_WRITE, providerClass, resolutionStrategy}); + opts.add(new Object[] {HoodieTableType.MERGE_ON_READ, providerClass, resolutionStrategy}); + } + } + return opts; + } + @ParameterizedTest @MethodSource("providerClassResolutionStrategyAndTableType") public void testMultiWriterWithAsyncTableServicesWithConflict(HoodieTableType tableType, Class> providerClass, @@ -586,6 +587,9 @@ public void testMultiWriterWithAsyncTableServicesWithConflict(HoodieTableType ta // Since the concurrent modifications went in, this upsert has // to fail + // ----- This line error out, go to the assertThrows API and set breakpoint there, keep running the test for 20 times in debug mode should repro the issue + // to keep running a test, modify Run configuration of the test (right click the run button next to the test, choose the last option) -> modify options -> repeat -> run until stopped. + // the run test in debug mode. when there is a repro it will stop at the breakpoint assertThrows(HoodieWriteConflictException.class, () -> { createCommitWithUpserts(cfg, client1, thirdCommitTime, commitTimeBetweenPrevAndNew, newCommitTime, numRecords); });