diff --git a/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java b/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java
index 39379d43aba..94b49bd94b2 100644
--- a/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java
+++ b/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java
@@ -335,10 +335,13 @@ private static void verifyNoSubscribers(ControlPlaneRule rule) {
 
   // This test takes a long time because of the 16 sec timeout for non-existent resource
   @Test
-  public void connect_then_mainServerDown_fallbackServerUp() throws InterruptedException {
+  public void connect_then_mainServerDown_fallbackServerUp() throws Exception {
     mainXdsServer.restartXdsServer();
     fallbackServer.restartXdsServer();
-    xdsClient = xdsClientPool.getObject();
+    XdsClientImpl xdsClient = CommonBootstrapperTestUtils.createXdsClient(
+        new GrpcBootstrapperImpl().bootstrap(defaultBootstrapOverride()),
+        DEFAULT_XDS_TRANSPORT_FACTORY, fakeClock, new ExponentialBackoffPolicy.Provider(),
+        MessagePrinter.INSTANCE, xdsClientMetricReporter);
 
     xdsClient.watchXdsResource(XdsListenerResource.getInstance(), MAIN_SERVER, ldsWatcher);
 
@@ -349,7 +352,12 @@ public void connect_then_mainServerDown_fallbackServerUp() throws InterruptedExc
     verify(rdsWatcher, timeout(5000)).onChanged(any());
 
     mainXdsServer.getServer().shutdownNow();
-    TimeUnit.SECONDS.sleep(5); // TODO(lsafran) Use FakeClock so test runs faster
+    // Sleep for the ADS stream disconnect to be processed and for the retry to fail. Between those
+    // two sleeps we need the fakeClock to progress by 1 second to restart the ADS stream.
+    for (int i = 0; i < 5; i++) {
+      fakeClock.forwardTime(1000, TimeUnit.MILLISECONDS);
+      TimeUnit.SECONDS.sleep(1);
+    }
 
     // Shouldn't do fallback since all watchers are loaded
     verify(ldsWatcher, never()).onChanged(
@@ -372,7 +380,7 @@ public void connect_then_mainServerDown_fallbackServerUp() throws InterruptedExc
         XdsListenerResource.LdsUpdate.forApiListener(FALLBACK_HTTP_CONNECTION_MANAGER));
     verify(ldsWatcher2, timeout(5000)).onChanged(
         XdsListenerResource.LdsUpdate.forApiListener(FALLBACK_HTTP_CONNECTION_MANAGER));
-    verify(cdsWatcher, timeout(16000)).onChanged(any());
+    verify(cdsWatcher, timeout(5000)).onChanged(any());
 
     xdsClient.watchXdsResource(
         XdsRouteConfigureResource.getInstance(), FALLBACK_RDS_NAME, rdsWatcher3);
@@ -381,7 +389,10 @@ public void connect_then_mainServerDown_fallbackServerUp() throws InterruptedExc
     // Test that resource defined in main but not fallback is handled correctly
     xdsClient.watchXdsResource(
         XdsClusterResource.getInstance(), CLUSTER_NAME, cdsWatcher2);
-    verify(cdsWatcher2, timeout(16000)).onResourceDoesNotExist(eq(CLUSTER_NAME));
+    verify(cdsWatcher2, never()).onResourceDoesNotExist(eq(CLUSTER_NAME));
+    fakeClock.forwardTime(15000, TimeUnit.MILLISECONDS); // Does not exist timer
+    verify(cdsWatcher2, timeout(5000)).onResourceDoesNotExist(eq(CLUSTER_NAME));
+    xdsClient.shutdown();
   }
 
   @Test
diff --git a/xds/src/test/java/io/grpc/xds/client/CommonBootstrapperTestUtils.java b/xds/src/test/java/io/grpc/xds/client/CommonBootstrapperTestUtils.java
index 27a0d4ba1d9..f3de4549ba9 100644
--- a/xds/src/test/java/io/grpc/xds/client/CommonBootstrapperTestUtils.java
+++ b/xds/src/test/java/io/grpc/xds/client/CommonBootstrapperTestUtils.java
@@ -23,7 +23,6 @@
 import io.grpc.internal.BackoffPolicy;
 import io.grpc.internal.FakeClock;
 import io.grpc.internal.JsonParser;
-import io.grpc.internal.TimeProvider;
 import io.grpc.xds.client.Bootstrapper.ServerInfo;
 import io.grpc.xds.internal.security.CommonTlsContextTestsUtil;
 import io.grpc.xds.internal.security.TlsContextManagerImpl;
@@ -32,28 +31,12 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 
 public class CommonBootstrapperTestUtils {
   private static final ChannelCredentials CHANNEL_CREDENTIALS = InsecureChannelCredentials.create();
   private static final String SERVER_URI_CUSTOM_AUTHORITY = "trafficdirector2.googleapis.com";
   private static final String SERVER_URI_EMPTY_AUTHORITY = "trafficdirector3.googleapis.com";
-
-  private static final long TIME_INCREMENT = TimeUnit.SECONDS.toNanos(1);
-
-  /** Fake time provider increments time TIME_INCREMENT each call. */
-  private static TimeProvider newTimeProvider() {
-    return new TimeProvider() {
-      private long count;
-
-      @Override
-      public long currentTimeNanos() {
-        return ++count * TIME_INCREMENT;
-      }
-    };
-  }
-
   private static final String FILE_WATCHER_CONFIG = "{\"path\": \"/etc/secret/certs\"}";
   private static final String MESHCA_CONFIG =
       "{\n"
@@ -183,14 +166,28 @@ public static XdsClientImpl createXdsClient(List<String> serverUris,
                                               BackoffPolicy.Provider backoffPolicyProvider,
                                               MessagePrettyPrinter messagePrinter,
                                               XdsClientMetricReporter xdsClientMetricReporter) {
-    Bootstrapper.BootstrapInfo bootstrapInfo = buildBootStrap(serverUris);
+    return createXdsClient(
+        buildBootStrap(serverUris),
+        xdsTransportFactory,
+        fakeClock,
+        backoffPolicyProvider,
+        messagePrinter,
+        xdsClientMetricReporter);
+  }
+
+  public static XdsClientImpl createXdsClient(Bootstrapper.BootstrapInfo bootstrapInfo,
+                                              XdsTransportFactory xdsTransportFactory,
+                                              FakeClock fakeClock,
+                                              BackoffPolicy.Provider backoffPolicyProvider,
+                                              MessagePrettyPrinter messagePrinter,
+                                              XdsClientMetricReporter xdsClientMetricReporter) {
     return new XdsClientImpl(
         xdsTransportFactory,
         bootstrapInfo,
         fakeClock.getScheduledExecutorService(),
         backoffPolicyProvider,
         fakeClock.getStopwatchSupplier(),
-        newTimeProvider(),
+        fakeClock.getTimeProvider(),
         messagePrinter,
         new TlsContextManagerImpl(bootstrapInfo),
         xdsClientMetricReporter);