Skip to content

Commit

Permalink
[#6270] (Improvement) unified cache framework (#6271)
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

Unified cache framework

### Why are the changes needed?

Fix: #6270

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

N/A
  • Loading branch information
sunxiaojian authored Jan 23, 2025
1 parent 4c5e9db commit 8813d15
Show file tree
Hide file tree
Showing 5 changed files with 9 additions and 6 deletions.
1 change: 1 addition & 0 deletions spark-connector/spark-common/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
dependencies {
implementation(project(":catalogs:catalog-common"))
implementation(libs.guava)
implementation(libs.caffeine)

compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
*/
package org.apache.gravitino.spark.connector.catalog;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.client.GravitinoClient;
import org.slf4j.Logger;
Expand All @@ -42,7 +41,7 @@ public class GravitinoCatalogManager {
private GravitinoCatalogManager(Supplier<GravitinoClient> clientBuilder) {
this.gravitinoClient = clientBuilder.get();
// Will not evict catalog by default
this.gravitinoCatalogs = CacheBuilder.newBuilder().build();
this.gravitinoCatalogs = Caffeine.newBuilder().build();
}

public static GravitinoCatalogManager create(Supplier<GravitinoClient> clientBuilder) {
Expand All @@ -69,8 +68,8 @@ public void close() {

public Catalog getGravitinoCatalogInfo(String name) {
try {
return gravitinoCatalogs.get(name, () -> loadCatalog(name));
} catch (ExecutionException e) {
return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName));
} catch (Exception e) {
LOG.error(String.format("Load catalog %s failed", name), e);
throw new RuntimeException(e);
}
Expand Down
1 change: 1 addition & 0 deletions spark-connector/v3.3/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
relocate("com.google", "org.apache.gravitino.shaded.com.google")
relocate("google", "org.apache.gravitino.shaded.google")
relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc")
relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
}

publishing {
Expand Down
1 change: 1 addition & 0 deletions spark-connector/v3.4/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
relocate("com.google", "org.apache.gravitino.shaded.com.google")
relocate("google", "org.apache.gravitino.shaded.google")
relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc")
relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
}

publishing {
Expand Down
1 change: 1 addition & 0 deletions spark-connector/v3.5/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
relocate("com.google", "org.apache.gravitino.shaded.com.google")
relocate("google", "org.apache.gravitino.shaded.google")
relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc")
relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
}

publishing {
Expand Down

0 comments on commit 8813d15

Please sign in to comment.