diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index dc8acd60678..fc2daf2159d 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -92,7 +92,7 @@ jobs: ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: authorizations-integrate-test-reports-${{ matrix.java-version }} diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml index b15c5d226ca..2932d56cb39 100644 --- a/.github/workflows/backend-integration-test-action.yml +++ b/.github/workflows/backend-integration-test-action.yml @@ -64,7 +64,7 @@ jobs: -x :authorizations:authorization-chain:test -x :authorizations:authorization-ranger:test - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e60f77ba67..c1a27c5f71f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,7 +90,7 @@ jobs: ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false - name: Upload unit tests report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: unit test report @@ -129,7 +129,7 @@ jobs: run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build - name: Upload unit tests report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: unit test report diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml index db805fe6da9..51797c436b3 100644 --- a/.github/workflows/cron-integration-test.yml +++ b/.github/workflows/cron-integration-test.yml @@ -87,7 +87,7 @@ jobs: ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} with: name: integrate test reports diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index e5f1f699de1..337504170ce 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -32,6 +32,11 @@ on: description: 'Publish Docker token' required: true type: string + publish-latest-tag: + description: 'Whether to update the latest tag. This operation is only applicable to official releases and should not be used for Release Candidate (RC).' + required: false + type: boolean + default: false jobs: publish-docker-image: @@ -83,6 +88,12 @@ jobs: echo "image_type=iceberg-rest-server" >> $GITHUB_ENV echo "image_name=apache/gravitino-iceberg-rest" >> $GITHUB_ENV fi + + if [ "${{ github.event.inputs.publish-latest-tag }}" == "true" ]; then + echo "publish_latest=true" >> $GITHUB_ENV + else + echo "publish_latest=false" >> $GITHUB_ENV + fi - name: Check publish Docker token run: | @@ -115,8 +126,16 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/hostedtoolcache/CodeQL - if [[ "${image_type}" == "gravitino" || "${image_type}" == "iceberg-rest-server" ]]; then - ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.version }} --latest + if [[ -n "${tag_name}" ]]; then + full_tag_name="${tag_name}-${{ github.event.inputs.version }}" + else + full_tag_name="${{ github.event.inputs.version }}" + fi + + if [[ "${publish_latest}" == "true" ]]; then + echo "Publish tag ${full_tag_name}, and update latest too." + ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} --latest else - ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag "${tag_name}-${{ github.event.inputs.version }}" - fi \ No newline at end of file + echo "Publish tag ${full_tag_name}." + ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} + fi diff --git a/.github/workflows/flink-integration-test-action.yml b/.github/workflows/flink-integration-test-action.yml index f22308c1951..d91de4064e3 100644 --- a/.github/workflows/flink-integration-test-action.yml +++ b/.github/workflows/flink-integration-test-action.yml @@ -52,7 +52,7 @@ jobs: ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: flink-connector-integrate-test-reports-${{ inputs.java-version }} diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 2dd2bee6474..046656fd9b1 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -88,7 +88,7 @@ jobs: ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: integrate-test-reports-${{ matrix.java-version }} diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml index 4fe7b66e09d..f1e97e5e88d 100644 --- a/.github/workflows/gvfs-fuse-build-test.yml +++ b/.github/workflows/gvfs-fuse-build-test.yml @@ -88,7 +88,7 @@ jobs: dev/ci/util_free_space.sh - name: Upload tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }} diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 546aa928584..5d997bba08d 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -82,7 +82,7 @@ jobs: done - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} with: name: integrate test reports diff --git a/.github/workflows/spark-integration-test-action.yml b/.github/workflows/spark-integration-test-action.yml index 873877bc29c..30c7ddd8eb3 100644 --- a/.github/workflows/spark-integration-test-action.yml +++ b/.github/workflows/spark-integration-test-action.yml @@ -63,7 +63,7 @@ jobs: ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: spark-connector-integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }} diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml index fb881a732af..cc383622831 100644 --- a/.github/workflows/trino-integration-test.yml +++ b/.github/workflows/trino-integration-test.yml @@ -90,7 +90,7 @@ jobs: trino-connector/integration-test/trino-test-tools/run_test.sh - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: name: trino-connector-integrate-test-reports-${{ matrix.java-version }} diff --git a/.gitignore b/.gitignore index eae3d3c952c..eb0adf56118 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ replay_pid* **/.gradle **/.idea !/.idea/icon.svg +!.idea/vcs.xml **/build gen **/.DS_Store diff --git a/api/src/main/java/org/apache/gravitino/MetadataObjects.java b/api/src/main/java/org/apache/gravitino/MetadataObjects.java index 557ccdefc49..e96b6e7e4a6 100644 --- a/api/src/main/java/org/apache/gravitino/MetadataObjects.java +++ b/api/src/main/java/org/apache/gravitino/MetadataObjects.java @@ -126,6 +126,7 @@ public static MetadataObject parent(MetadataObject object) { case TABLE: case FILESET: case TOPIC: + case MODEL: parentType = MetadataObject.Type.SCHEMA; break; case SCHEMA: diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java index 9720b813fa5..a1963f64539 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java @@ -122,8 +122,13 @@ public int hashCode() { @Override public String toString() { - return "MetadataObject: [fullName=" + fullName() + "], [path=" + path == null - ? "null" - : path + "], [type=" + type + "]"; + String strPath = path == null ? "null" : path; + return "MetadataObject: [fullName=" + + fullName() + + "], [path=" + + strPath + + "], [type=" + + type + + "]"; } } diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index d0a1b0897ec..2d74bd050e3 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -110,8 +110,13 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime @Override public Boolean onRoleCreated(Role role) throws AuthorizationPluginException { List sqls = getCreateRoleSQL(role.name()); + boolean createdNewly = false; for (String sql : sqls) { - executeUpdateSQL(sql, "already exists"); + createdNewly = executeUpdateSQL(sql, "already exists"); + } + + if (!createdNewly) { + return true; } if (role.securableObjects() != null) { @@ -140,7 +145,6 @@ public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException { @Override public Boolean onRoleUpdated(Role role, RoleChange... changes) throws AuthorizationPluginException { - onRoleCreated(role); for (RoleChange change : changes) { if (change instanceof RoleChange.AddSecurableObject) { SecurableObject object = ((RoleChange.AddSecurableObject) change).getSecurableObject(); @@ -381,14 +385,15 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode()); } - public void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public boolean executeUpdateSQL(String sql, String ignoreErrorMsg) { try (final Connection connection = getConnection()) { try (final Statement statement = connection.createStatement()) { statement.executeUpdate(sql); + return true; } } catch (SQLException se) { if (ignoreErrorMsg != null && se.getMessage().contains(ignoreErrorMsg)) { - return; + return false; } LOG.error("JDBC authorization plugin exception: ", se); throw toAuthorizationPluginException(se); diff --git a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java index 3f604b5f389..d2a2291a356 100644 --- a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java @@ -47,4 +47,31 @@ public void PathBasedMetadataObjectNotEquals() { Assertions.assertNotEquals(pathBasedMetadataObject1, pathBasedMetadataObject2); } + + @Test + void testToString() { + PathBasedMetadataObject pathBasedMetadataObject1 = + new PathBasedMetadataObject("parent", "name", "path", PathBasedMetadataObject.Type.PATH); + Assertions.assertEquals( + "MetadataObject: [fullName=parent.name], [path=path], [type=PATH]", + pathBasedMetadataObject1.toString()); + + PathBasedMetadataObject pathBasedMetadataObject2 = + new PathBasedMetadataObject("parent", "name", null, PathBasedMetadataObject.Type.PATH); + Assertions.assertEquals( + "MetadataObject: [fullName=parent.name], [path=null], [type=PATH]", + pathBasedMetadataObject2.toString()); + + PathBasedMetadataObject pathBasedMetadataObject3 = + new PathBasedMetadataObject(null, "name", null, PathBasedMetadataObject.Type.PATH); + Assertions.assertEquals( + "MetadataObject: [fullName=name], [path=null], [type=PATH]", + pathBasedMetadataObject3.toString()); + + PathBasedMetadataObject pathBasedMetadataObject4 = + new PathBasedMetadataObject(null, "name", "path", PathBasedMetadataObject.Type.PATH); + Assertions.assertEquals( + "MetadataObject: [fullName=name], [path=path], [type=PATH]", + pathBasedMetadataObject4.toString()); + } } diff --git a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java index 6f9f7e29c37..8d3788617cd 100644 --- a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java @@ -74,9 +74,10 @@ public List getSetOwnerSQL( return Collections.emptyList(); } - public void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public boolean executeUpdateSQL(String sql, String ignoreErrorMsg) { Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql); currentSQLIndex++; + return true; } }; @@ -148,23 +149,21 @@ public void testPermissionManagement() { // Test metalake object and different role change resetSQLIndex(); - expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp"); + expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE *.* TO ROLE tmp"); SecurableObject metalakeObject = SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.SelectTable.allow())); RoleChange roleChange = RoleChange.addSecurableObject("tmp", metalakeObject); plugin.onRoleUpdated(role, roleChange); resetSQLIndex(); - expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "REVOKE SELECT ON TABLE *.* FROM ROLE tmp"); + expectSQLs = Lists.newArrayList("REVOKE SELECT ON TABLE *.* FROM ROLE tmp"); roleChange = RoleChange.removeSecurableObject("tmp", metalakeObject); plugin.onRoleUpdated(role, roleChange); resetSQLIndex(); expectSQLs = Lists.newArrayList( - "CREATE ROLE tmp", - "REVOKE SELECT ON TABLE *.* FROM ROLE tmp", - "GRANT CREATE ON TABLE *.* TO ROLE tmp"); + "REVOKE SELECT ON TABLE *.* FROM ROLE tmp", "GRANT CREATE ON TABLE *.* TO ROLE tmp"); SecurableObject newMetalakeObject = SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.CreateTable.allow())); roleChange = RoleChange.updateSecurableObject("tmp", metalakeObject, newMetalakeObject); @@ -175,7 +174,7 @@ public void testPermissionManagement() { SecurableObject catalogObject = SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.SelectTable.allow())); roleChange = RoleChange.addSecurableObject("tmp", catalogObject); - expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp"); + expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE *.* TO ROLE tmp"); plugin.onRoleUpdated(role, roleChange); // Test schema object @@ -184,8 +183,7 @@ public void testPermissionManagement() { SecurableObjects.ofSchema( catalogObject, "schema", Lists.newArrayList(Privileges.SelectTable.allow())); roleChange = RoleChange.addSecurableObject("tmp", schemaObject); - expectSQLs = - Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.* TO ROLE tmp"); + expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE schema.* TO ROLE tmp"); plugin.onRoleUpdated(role, roleChange); // Test table object @@ -194,8 +192,18 @@ public void testPermissionManagement() { SecurableObjects.ofTable( schemaObject, "table", Lists.newArrayList(Privileges.SelectTable.allow())); roleChange = RoleChange.addSecurableObject("tmp", tableObject); - expectSQLs = - Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.table TO ROLE tmp"); + expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE schema.table TO ROLE tmp"); + plugin.onRoleUpdated(role, roleChange); + + // Test the role with objects + resetSQLIndex(); + role = + RoleEntity.builder() + .withId(-1L) + .withName("tmp") + .withSecurableObjects(Lists.newArrayList(tableObject)) + .withAuditInfo(AuditInfo.EMPTY) + .build(); plugin.onRoleUpdated(role, roleChange); } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java index df1340c947e..92c5d18a129 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java @@ -33,6 +33,7 @@ public class IcebergPropertiesUtils { // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will // change it to `catalogType` automatically and pass it to Iceberg. public static final Map GRAVITINO_CONFIG_TO_ICEBERG; + public static final Map ICEBERG_CATALOG_CONFIG_TO_GRAVITINO; static { Map map = new HashMap(); @@ -65,6 +66,14 @@ public class IcebergPropertiesUtils { AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, IcebergConstants.ICEBERG_ADLS_STORAGE_ACCOUNT_KEY); GRAVITINO_CONFIG_TO_ICEBERG = Collections.unmodifiableMap(map); + + Map icebergCatalogConfigToGravitino = new HashMap<>(); + map.forEach( + (key, value) -> { + icebergCatalogConfigToGravitino.put(value, key); + }); + ICEBERG_CATALOG_CONFIG_TO_GRAVITINO = + Collections.unmodifiableMap(icebergCatalogConfigToGravitino); } /** diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 6a8b815ab97..b5593c6f7e4 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -158,6 +158,8 @@ tasks { exclude("guava-*.jar") exclude("log4j-*.jar") exclude("slf4j-*.jar") + // Exclude the following jars to avoid conflict with the jars in authorization-gcp + exclude("protobuf-java-*.jar") } into("$rootDir/distribution/package/catalogs/hive/libs") } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/AreYouSure.java b/clients/cli/src/main/java/org/apache/gravitino/cli/AreYouSure.java index a0893dbad4f..a6d1450c1bf 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/AreYouSure.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/AreYouSure.java @@ -32,16 +32,20 @@ public class AreYouSure { * @return {@code true} if the action is to continue {@code false} otherwise. */ public static boolean really(boolean force) { - Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8.name()); - /* force option for scripting */ if (force) { return true; } - System.out.println( - "This command could result in data loss or other issues. Are you sure you want to do this? (Y/N)"); - String answer = scanner.next(); - return answer.equals("Y"); + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8.name())) { + System.out.println( + "This command could result in data loss or other issues. Are you sure you want to do this? (Y/N)"); + String answer = scanner.next(); + return "Y".equals(answer); + } catch (Exception e) { + System.err.println("Error while reading user input: " + e.getMessage()); + Main.exit(-1); + } + return false; } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/RoleCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/RoleCommandHandler.java index cf48783b8dc..327ef21fdd8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/RoleCommandHandler.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/RoleCommandHandler.java @@ -137,11 +137,19 @@ url, ignore, metalake, getOneRole(), new FullName(line), privileges) } private void handleRevokeCommand() { - gravitinoCommandLine - .newRevokePrivilegesFromRole( - url, ignore, metalake, getOneRole(), new FullName(line), privileges) - .validate() - .handle(); + boolean removeAll = line.hasOption(GravitinoOptions.ALL); + if (removeAll) { + gravitinoCommandLine + .newRevokeAllPrivileges(url, ignore, metalake, getOneRole(), new FullName(line)) + .validate() + .handle(); + } else { + gravitinoCommandLine + .newRevokePrivilegesFromRole( + url, ignore, metalake, getOneRole(), new FullName(line), privileges) + .validate() + .handle(); + } } private String getOneRole() { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index 498a0060cbd..b229ef16aa3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -97,6 +97,7 @@ import org.apache.gravitino.cli.commands.RemoveTableProperty; import org.apache.gravitino.cli.commands.RemoveTagProperty; import org.apache.gravitino.cli.commands.RemoveTopicProperty; +import org.apache.gravitino.cli.commands.RevokeAllPrivileges; import org.apache.gravitino.cli.commands.RevokePrivilegesFromRole; import org.apache.gravitino.cli.commands.RoleAudit; import org.apache.gravitino.cli.commands.RoleDetails; @@ -901,6 +902,11 @@ protected RevokePrivilegesFromRole newRevokePrivilegesFromRole( return new RevokePrivilegesFromRole(url, ignore, metalake, role, entity, privileges); } + protected RevokeAllPrivileges newRevokeAllPrivileges( + String url, boolean ignore, String metalake, String role, FullName entity) { + return new RevokeAllPrivileges(url, ignore, metalake, role, entity); + } + protected MetalakeEnable newMetalakeEnable( String url, boolean ignore, String metalake, boolean enableAllCatalogs) { return new MetalakeEnable(url, ignore, metalake, enableAllCatalogs); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java index 8485a587560..a815d6ba14d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java @@ -68,7 +68,7 @@ public OwnerDetails( /** Displays the owner of an entity. */ @Override public void handle() { - Optional owner = null; + Optional owner = Optional.empty(); MetadataObject metadata = MetadataObjects.parse(entity, entityType); try { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokeAllPrivileges.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokeAllPrivileges.java new file mode 100644 index 00000000000..3167c4b6c37 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokeAllPrivileges.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NoSuchRoleException; + +/** Revokes all privileges from a role to an entity or all entities. */ +public class RevokeAllPrivileges extends MetadataCommand { + + protected final String metalake; + protected final String role; + protected final FullName entity; + + /** + * Revokes all privileges from a role to an entity or all entities. + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param role The name of the role. + * @param entity The name of the entity. + */ + public RevokeAllPrivileges( + String url, boolean ignoreVersions, String metalake, String role, FullName entity) { + super(url, ignoreVersions); + this.metalake = metalake; + this.role = role; + this.entity = entity; + } + + /** Revokes all privileges from a role to an entity or all entities. */ + @Override + public void handle() { + List matchedObjects; + Map> revokedPrivileges = Maps.newHashMap(); + + try { + GravitinoClient client = buildClient(metalake); + matchedObjects = getMatchedObjects(client); + + for (SecurableObject securableObject : matchedObjects) { + String objectFullName = securableObject.fullName(); + Set privileges = new HashSet<>(securableObject.privileges()); + MetadataObject metadataObject = constructMetadataObject(entity, client); + client.revokePrivilegesFromRole(role, metadataObject, privileges); + + revokedPrivileges.put(objectFullName, privileges); + } + } catch (NoSuchMetalakeException e) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (NoSuchRoleException e) { + exitWithError(ErrorMessages.UNKNOWN_ROLE); + } catch (Exception e) { + exitWithError(e.getMessage()); + } + + if (revokedPrivileges.isEmpty()) outputRevokedPrivileges("No privileges revoked."); + outputRevokedPrivileges(revokedPrivileges); + } + + private List getMatchedObjects(GravitinoClient client) { + Role gRole = client.getRole(role); + return gRole.securableObjects().stream() + .filter(s -> s.fullName().equals(entity.getName())) + .collect(Collectors.toList()); + } + + private void outputRevokedPrivileges(Map> revokedPrivileges) { + List revokedInfoList = Lists.newArrayList(); + + for (Map.Entry> entry : revokedPrivileges.entrySet()) { + List revokedPrivilegesList = + entry.getValue().stream().map(Privilege::simpleString).collect(Collectors.toList()); + revokedInfoList.add(entry.getKey() + ": " + COMMA_JOINER.join(revokedPrivilegesList)); + } + + System.out.println("Revoked privileges:"); + for (String info : revokedInfoList) { + System.out.println(info); + } + } + + private void outputRevokedPrivileges(String message) { + System.out.println(message); + } + + /** + * verify the arguments. + * + * @return Returns itself via argument validation, otherwise exits. + */ + @Override + public Command validate() { + if (!entity.hasName()) exitWithError(ErrorMessages.MISSING_NAME); + return super.validate(); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java index fbf273ce0d8..5907c494c05 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java @@ -87,17 +87,13 @@ public void handle() { MetadataObject metadataObject = constructMetadataObject(entity, client); client.revokePrivilegesFromRole(role, metadataObject, privilegesSet); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (NoSuchMetadataObjectException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", privileges); diff --git a/clients/cli/src/main/resources/role_help.txt b/clients/cli/src/main/resources/role_help.txt index d0838cfa403..6c51980d1c5 100644 --- a/clients/cli/src/main/resources/role_help.txt +++ b/clients/cli/src/main/resources/role_help.txt @@ -42,3 +42,6 @@ gcli role grant --name catalog_postgres --role admin --privilege create_table mo Revoke a privilege gcli role revoke --metalake metalake_demo --name catalog_postgres --role admin --privilege create_table modify_table + +Revoke all privileges +gcli role revoke --metalake metalake_demo --name catalog_postgres --role admin --all \ No newline at end of file diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestAreYouSure.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestAreYouSure.java new file mode 100644 index 00000000000..78a1c2d67d4 --- /dev/null +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestAreYouSure.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestAreYouSure { + + @Test + void testCommandWithForce() { + Assertions.assertTrue(AreYouSure.really(true)); + } + + @Test + void testCommandWithInputY() { + ByteArrayInputStream inputStream = + new ByteArrayInputStream("Y".getBytes(StandardCharsets.UTF_8)); + System.setIn(inputStream); + + Assertions.assertTrue(AreYouSure.really(false)); + } + + @Test + void testCommandWithInputN() { + ByteArrayInputStream inputStream = + new ByteArrayInputStream("N".getBytes(StandardCharsets.UTF_8)); + System.setIn(inputStream); + + Assertions.assertFalse(AreYouSure.really(false)); + } + + @Test + void testCommandWithInputInvalid() { + ByteArrayInputStream inputStream = + new ByteArrayInputStream("Invalid".getBytes(StandardCharsets.UTF_8)); + System.setIn(inputStream); + + Assertions.assertFalse(AreYouSure.really(false)); + } +} diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestModelCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestModelCommands.java index b83cc3c3136..4f1507cc703 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestModelCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestModelCommands.java @@ -89,11 +89,7 @@ void testListModelCommand() { doReturn(mockList) .when(commandLine) .newListModel( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("catalog"), - eq("schema")); + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema"); doReturn(mockList).when(mockList).validate(); commandLine.handleCommandLine(); verify(mockList).handle(); @@ -112,12 +108,7 @@ void testListModelCommandWithoutCatalog() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) - .newListModel( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - isNull(), - isNull()); + .newListModel(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals( ErrorMessages.MISSING_NAME @@ -141,12 +132,7 @@ void testListModelCommandWithoutSchema() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) - .newListModel( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("catalog"), - isNull()); + .newListModel(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals( ErrorMessages.MALFORMED_NAME @@ -171,12 +157,7 @@ void testModelDetailsCommand() { doReturn(mockList) .when(commandLine) .newModelDetails( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("catalog"), - eq("schema"), - eq("model")); + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", "model"); doReturn(mockList).when(mockList).validate(); commandLine.handleCommandLine(); verify(mockList).handle(); @@ -197,12 +178,7 @@ void testModelDetailsCommandWithoutCatalog() { verify(commandLine, never()) .newModelDetails( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - isNull(), - isNull(), - isNull()); + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals( ErrorMessages.MISSING_NAME @@ -230,12 +206,7 @@ void testModelDetailsCommandWithoutSchema() { verify(commandLine, never()) .newModelDetails( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("catalog"), - isNull(), - isNull()); + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals( ErrorMessages.MALFORMED_NAME @@ -261,12 +232,7 @@ void testModelDetailsCommandWithoutModel() { verify(commandLine, never()) .newModelDetails( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("catalog"), - eq("schema"), - isNull()); + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals( ErrorMessages.MALFORMED_NAME diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java index 12f617380ca..f3928fba28f 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java @@ -21,8 +21,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -160,11 +158,11 @@ void testOwnerDetailsCommandWithoutName() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newOwnerDetails( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq(null), - eq(CommandEntities.CATALOG)); + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + null, + CommandEntities.CATALOG); String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals(ErrorMessages.MISSING_NAME, errOutput); @@ -188,13 +186,13 @@ void testSetOwnerUserCommandWithoutUserAndGroup() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newSetOwner( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq("postgres"), - eq(CommandEntities.CATALOG), - isNull(), - eq(false)); + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "postgres", + CommandEntities.CATALOG, + null, + false); String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals(ErrorMessages.INVALID_SET_COMMAND, errOutput); } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java index 529979582ff..be463019fd3 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java @@ -34,11 +34,14 @@ import java.io.PrintStream; import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.gravitino.cli.commands.CreateRole; import org.apache.gravitino.cli.commands.DeleteRole; import org.apache.gravitino.cli.commands.GrantPrivilegesToRole; import org.apache.gravitino.cli.commands.ListRoles; +import org.apache.gravitino.cli.commands.RevokeAllPrivileges; import org.apache.gravitino.cli.commands.RevokePrivilegesFromRole; import org.apache.gravitino.cli.commands.RoleAudit; import org.apache.gravitino.cli.commands.RoleDetails; @@ -48,6 +51,7 @@ class TestRoleCommands { private CommandLine mockCommandLine; + private Options options; private Options mockOptions; private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); @@ -58,6 +62,7 @@ class TestRoleCommands { void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + options = new GravitinoOptions().options(); System.setOut(new PrintStream(outContent)); System.setErr(new PrintStream(errContent)); } @@ -99,6 +104,7 @@ void testRoleDetailsCommand() { doReturn(mockDetails) .when(commandLine) .newRoleDetails(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "admin"); + doReturn(mockDetails).when(mockDetails).validate(); commandLine.handleCommandLine(); verify(mockDetails).handle(); @@ -179,10 +185,10 @@ void testCreateRolesCommand() { doReturn(mockCreate) .when(commandLine) .newCreateRole( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq("metalake_demo"), - eq(new String[] {"admin", "engineer", "scientist"})); + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + new String[] {"admin", "engineer", "scientist"}); doReturn(mockCreate).when(mockCreate).validate(); commandLine.handleCommandLine(); verify(mockCreate).handle(); @@ -229,11 +235,11 @@ void testDeleteRolesCommand() { doReturn(mockDelete) .when(commandLine) .newDeleteRole( - eq(GravitinoCommandLine.DEFAULT_URL), - eq(false), - eq(false), - eq("metalake_demo"), - eq(new String[] {"admin", "engineer", "scientist"})); + GravitinoCommandLine.DEFAULT_URL, + false, + false, + "metalake_demo", + new String[] {"admin", "engineer", "scientist"}); doReturn(mockDelete).when(mockDelete).validate(); commandLine.handleCommandLine(); verify(mockDelete).handle(); @@ -333,6 +339,51 @@ void testRevokePrivilegesFromRole() { verify(mockRevoke).handle(); } + @Test + void testRevokeAllPrivilegesFromRole() { + RevokeAllPrivileges mockRevoke = mock(RevokeAllPrivileges.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); + when(mockCommandLine.hasOption(GravitinoOptions.ALL)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.REVOKE)); + doReturn(mockRevoke) + .when(commandLine) + .newRevokeAllPrivileges( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("admin"), + any()); + doReturn(mockRevoke).when(mockRevoke).validate(); + commandLine.handleCommandLine(); + verify(mockRevoke).handle(); + } + + @Test + void testRevokeAllPrivilegesFromRoleWithoutName() throws ParseException { + Main.useExit = false; + String[] args = {"role", "revoke", "-m", "metalake_demo", "--role", "admin", "--all"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + + RevokeAllPrivileges spyRevokeAllPrivileges = + spy( + new RevokeAllPrivileges( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "admin", fullName)); + + assertThrows(RuntimeException.class, spyRevokeAllPrivileges::validate); + verify(spyRevokeAllPrivileges, never()).handle(); + String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(ErrorMessages.MISSING_NAME, errOutput); + } + @Test void testRevokePrivilegesFromRoleWithoutPrivileges() { Main.useExit = false; diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModel.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModel.java index 2d356b712fe..d4c0c24f1a9 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModel.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModel.java @@ -18,20 +18,35 @@ */ package org.apache.gravitino.client; +import com.google.common.collect.Lists; +import java.util.List; import java.util.Map; import org.apache.gravitino.Audit; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.Namespace; import org.apache.gravitino.authorization.SupportsRoles; import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.exceptions.NoSuchTagException; +import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; import org.apache.gravitino.model.Model; import org.apache.gravitino.tag.SupportsTags; +import org.apache.gravitino.tag.Tag; /** Represents a generic model. */ -class GenericModel implements Model { +class GenericModel implements Model, SupportsTags { private final ModelDTO modelDTO; - GenericModel(ModelDTO modelDTO) { + private final MetadataObjectTagOperations objectTagOperations; + + GenericModel(ModelDTO modelDTO, RESTClient restClient, Namespace modelNs) { this.modelDTO = modelDTO; + List modelFullName = + Lists.newArrayList(modelNs.level(1), modelNs.level(2), modelDTO.name()); + MetadataObject modelObject = MetadataObjects.of(modelFullName, MetadataObject.Type.MODEL); + this.objectTagOperations = + new MetadataObjectTagOperations(modelNs.level(0), modelObject, restClient); } @Override @@ -61,7 +76,7 @@ public int latestVersion() { @Override public SupportsTags supportsTags() { - throw new UnsupportedOperationException("Not supported yet."); + return this; } @Override @@ -86,4 +101,25 @@ public boolean equals(Object o) { public int hashCode() { return modelDTO.hashCode(); } + + @Override + public String[] listTags() { + return objectTagOperations.listTags(); + } + + @Override + public Tag[] listTagsInfo() { + return objectTagOperations.listTagsInfo(); + } + + @Override + public Tag getTag(String name) throws NoSuchTagException { + return objectTagOperations.getTag(name); + } + + @Override + public String[] associateTags(String[] tagsToAdd, String[] tagsToRemove) + throws TagAlreadyAssociatedException { + return objectTagOperations.associateTags(tagsToAdd, tagsToRemove); + } } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModelCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModelCatalog.java index 50e9eb246ac..8658a972596 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModelCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GenericModelCatalog.java @@ -97,7 +97,7 @@ public Model getModel(NameIdentifier ident) throws NoSuchModelException { ErrorHandlers.modelErrorHandler()); resp.validate(); - return new GenericModel(resp.getModel()); + return new GenericModel(resp.getModel(), restClient, modelFullNs); } @Override @@ -118,7 +118,7 @@ public Model registerModel(NameIdentifier ident, String comment, Map c.name().equals("col1")).findFirst().get(); + + // Create model catalog + String modelCatalogName = GravitinoITUtils.genRandomName("tag_it_model_catalog"); + Assertions.assertFalse(metalake.catalogExists(modelCatalogName)); + modelCatalog = + metalake.createCatalog( + modelCatalogName, Catalog.Type.MODEL, "comment", Collections.emptyMap()); + + // Create model schema + String modelSchemaName = GravitinoITUtils.genRandomName("tag_it_model_schema"); + Assertions.assertFalse(modelCatalog.asSchemas().schemaExists(modelSchemaName)); + modelSchema = + modelCatalog.asSchemas().createSchema(modelSchemaName, "comment", Collections.emptyMap()); + + // Create model + String modelName = GravitinoITUtils.genRandomName("tag_it_model"); + Assertions.assertFalse( + modelCatalog.asModelCatalog().modelExists(NameIdentifier.of(modelSchemaName, modelName))); + model = + modelCatalog + .asModelCatalog() + .registerModel( + NameIdentifier.of(modelSchemaName, modelName), "comment", Collections.emptyMap()); } @AfterAll @@ -115,6 +143,11 @@ public void tearDown() { relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name())); relationalCatalog.asSchemas().dropSchema(schema.name(), true); metalake.dropCatalog(relationalCatalog.name(), true); + + modelCatalog.asModelCatalog().deleteModel(NameIdentifier.of(modelSchema.name(), model.name())); + modelCatalog.asSchemas().dropSchema(modelSchema.name(), true); + metalake.dropCatalog(modelCatalog.name(), true); + client.dropMetalake(metalakeName, true); if (client != null) { @@ -653,4 +686,84 @@ public void testAssociateAndDeleteTags() { String[] associatedTags1 = relationalCatalog.supportsTags().listTags(); Assertions.assertArrayEquals(new String[] {tag2.name()}, associatedTags1); } + + @Test + public void testAssociateTagsToModel() { + Tag tag1 = + metalake.createTag( + GravitinoITUtils.genRandomName("tag_it_model_tag1"), + "comment1", + Collections.emptyMap()); + Tag tag2 = + metalake.createTag( + GravitinoITUtils.genRandomName("tag_it_model_tag2"), + "comment2", + Collections.emptyMap()); + Tag tag3 = + metalake.createTag( + GravitinoITUtils.genRandomName("tag_it_model_tag3"), + "comment3", + Collections.emptyMap()); + + // Associate tags to catalog + modelCatalog.supportsTags().associateTags(new String[] {tag1.name()}, null); + + // Associate tags to schema + modelSchema.supportsTags().associateTags(new String[] {tag2.name()}, null); + + // Associate tags to model + model.supportsTags().associateTags(new String[] {tag3.name()}, null); + + // Test list associated tags for model + String[] tags1 = model.supportsTags().listTags(); + Assertions.assertEquals(3, tags1.length); + Set tagNames = Sets.newHashSet(tags1); + Assertions.assertTrue(tagNames.contains(tag1.name())); + Assertions.assertTrue(tagNames.contains(tag2.name())); + Assertions.assertTrue(tagNames.contains(tag3.name())); + + // Test list associated tags with details for model + Tag[] tags2 = model.supportsTags().listTagsInfo(); + Assertions.assertEquals(3, tags2.length); + + Set nonInheritedTags = + Arrays.stream(tags2).filter(tag -> !tag.inherited().get()).collect(Collectors.toSet()); + Set inheritedTags = + Arrays.stream(tags2).filter(tag -> tag.inherited().get()).collect(Collectors.toSet()); + + Assertions.assertEquals(1, nonInheritedTags.size()); + Assertions.assertEquals(2, inheritedTags.size()); + Assertions.assertTrue(nonInheritedTags.contains(tag3)); + Assertions.assertTrue(inheritedTags.contains(tag1)); + Assertions.assertTrue(inheritedTags.contains(tag2)); + + // Test get associated tag for model + Tag resultTag1 = model.supportsTags().getTag(tag1.name()); + Assertions.assertEquals(tag1, resultTag1); + Assertions.assertTrue(resultTag1.inherited().get()); + + Tag resultTag2 = model.supportsTags().getTag(tag2.name()); + Assertions.assertEquals(tag2, resultTag2); + Assertions.assertTrue(resultTag2.inherited().get()); + + Tag resultTag3 = model.supportsTags().getTag(tag3.name()); + Assertions.assertEquals(tag3, resultTag3); + Assertions.assertFalse(resultTag3.inherited().get()); + + // Test get objects associated with tag + Assertions.assertEquals(1, tag1.associatedObjects().count()); + Assertions.assertEquals(modelCatalog.name(), tag1.associatedObjects().objects()[0].name()); + Assertions.assertEquals( + MetadataObject.Type.CATALOG, tag1.associatedObjects().objects()[0].type()); + + Assertions.assertEquals(1, tag2.associatedObjects().count()); + Assertions.assertEquals(modelSchema.name(), tag2.associatedObjects().objects()[0].name()); + Assertions.assertEquals( + MetadataObject.Type.SCHEMA, tag2.associatedObjects().objects()[0].type()); + + Assertions.assertEquals(1, tag3.associatedObjects().count()); + Assertions.assertEquals(model.name(), tag3.associatedObjects().objects()[0].name()); + Assertions.assertEquals( + MetadataObject.Type.MODEL, tag3.associatedObjects().objects()[0].type()); + } } diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index af6cfcd2d9f..29ec663e7d8 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -122,10 +122,10 @@ fun generatePypiProjectHomePage() { // relative path of the images in the how-to-use-python-client.md file is incorrect. We need // to fix the relative path of the images/markdown to the absolute path. val content = outputFile.readText() - val docsUrl = "https://datastrato.ai/docs/latest" + val docsUrl = "https://gravitino.apache.org/docs/latest" // Use regular expression to match the `[](./a/b/c.md?language=python)` or `[](./a/b/c.md#arg1)` link in the content - // Convert `[](./a/b/c.md?language=python)` to `[](https://datastrato.ai/docs/latest/a/b/c/language=python)` + // Convert `[](./a/b/c.md?language=python)` to `[](https://gravitino.apache.org/docs/latest/a/b/c/language=python)` val patternDocs = Regex("""(? val text = matchResult.groupValues[1] diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 499ba5cbf1f..60ffe3e83c8 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -29,6 +29,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; import org.apache.gravitino.GravitinoEnv; @@ -476,7 +477,7 @@ public static List getMetadataObjectLocation( Schema schema = GravitinoEnv.getInstance().schemaDispatcher().loadSchema(ident); if (schema.properties().containsKey(HiveConstants.LOCATION)) { String schemaLocation = schema.properties().get(HiveConstants.LOCATION); - if (schemaLocation != null && schemaLocation.isEmpty()) { + if (StringUtils.isNotBlank(schemaLocation)) { locations.add(schemaLocation); } else { LOG.warn("Schema %s location is not found", ident); @@ -497,7 +498,7 @@ public static List getMetadataObjectLocation( Table table = GravitinoEnv.getInstance().tableDispatcher().loadTable(ident); if (table.properties().containsKey(HiveConstants.LOCATION)) { String tableLocation = table.properties().get(HiveConstants.LOCATION); - if (tableLocation != null && tableLocation.isEmpty()) { + if (StringUtils.isNotBlank(tableLocation)) { locations.add(tableLocation); } else { LOG.warn("Table %s location is not found", ident); diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java index b602471c4d1..373785d539b 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java @@ -18,16 +18,25 @@ */ package org.apache.gravitino.authorization; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.util.List; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.Entity; +import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.CatalogDispatcher; +import org.apache.gravitino.catalog.TableDispatcher; import org.apache.gravitino.exceptions.IllegalNameIdentifierException; import org.apache.gravitino.exceptions.IllegalNamespaceException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.rel.Table; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; class TestAuthorizationUtils { @@ -210,4 +219,32 @@ void testFilteredSecurableObjects() { Assertions.assertTrue(filteredRole.securableObjects().contains(schema2Object)); Assertions.assertTrue(filteredRole.securableObjects().contains(table2Object)); } + + @Test + void testGetMetadataObjectLocation() throws IllegalAccessException { + CatalogDispatcher catalogDispatcher = Mockito.mock(CatalogDispatcher.class); + TableDispatcher tableDispatcher = Mockito.mock(TableDispatcher.class); + Catalog catalog = Mockito.mock(Catalog.class); + Table table = Mockito.mock(Table.class); + + Mockito.when(table.properties()).thenReturn(ImmutableMap.of("location", "gs://bucket/1")); + Mockito.when(catalog.provider()).thenReturn("hive"); + Mockito.when(catalogDispatcher.loadCatalog(Mockito.any())).thenReturn(catalog); + Mockito.when(tableDispatcher.loadTable(Mockito.any())).thenReturn(table); + + FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogDispatcher", catalogDispatcher, true); + FieldUtils.writeField(GravitinoEnv.getInstance(), "tableDispatcher", tableDispatcher, true); + + List locations = + AuthorizationUtils.getMetadataObjectLocation( + NameIdentifier.of("catalog", "schema", "table"), Entity.EntityType.TABLE); + Assertions.assertEquals(1, locations.size()); + Assertions.assertEquals("gs://bucket/1", locations.get(0)); + + locations = + AuthorizationUtils.getMetadataObjectLocation( + NameIdentifier.of("catalog", "schema", "fileset"), Entity.EntityType.TABLE); + Assertions.assertEquals(1, locations.size()); + Assertions.assertEquals("gs://bucket/1", locations.get(0)); + } } diff --git a/docs/assets/publish-docker-image.jpg b/docs/assets/publish-docker-image.jpg deleted file mode 100644 index ca22da1db96..00000000000 Binary files a/docs/assets/publish-docker-image.jpg and /dev/null differ diff --git a/docs/assets/publish-docker-image.png b/docs/assets/publish-docker-image.png new file mode 100644 index 00000000000..8085d2dc7de Binary files /dev/null and b/docs/assets/publish-docker-image.png differ diff --git a/docs/cli.md b/docs/cli.md index 7f6f190dd1b..cc710a20805 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -798,6 +798,12 @@ gcli role grant --name catalog_postgres --role admin --privilege create_table mo gcli role revoke --metalake metalake_demo --name catalog_postgres --role admin --privilege create_table modify_table ``` +### Revoke all privileges + +```bash +gcli role revoke --metalake metalake_demo --name catalog_postgres --role admin --all +``` + ### Topic commands #### Display a topic's details diff --git a/docs/flink-connector/flink-catalog-hive.md b/docs/flink-connector/flink-catalog-hive.md index ae55817067d..9fc9349e353 100644 --- a/docs/flink-connector/flink-catalog-hive.md +++ b/docs/flink-connector/flink-catalog-hive.md @@ -34,7 +34,7 @@ Supports most DDL and DML operations in Flink SQL, except such operations: ```sql // Suppose hive_a is the Hive catalog name managed by Gravitino -USE hive_a; +USE CATALOG hive_a; CREATE DATABASE IF NOT EXISTS mydatabase; USE mydatabase; diff --git a/docs/flink-connector/flink-catalog-iceberg.md b/docs/flink-connector/flink-catalog-iceberg.md index 54d7c0879fb..76142369f9c 100644 --- a/docs/flink-connector/flink-catalog-iceberg.md +++ b/docs/flink-connector/flink-catalog-iceberg.md @@ -32,11 +32,12 @@ To enable the Flink connector, you must download the Iceberg Flink runtime JAR a - `CREATE TABLE LIKE` clause ## SQL example + ```sql -- Suppose iceberg_a is the Iceberg catalog name managed by Gravitino -USE iceberg_a; +USE CATALOG iceberg_a; CREATE DATABASE IF NOT EXISTS mydatabase; USE mydatabase; @@ -59,15 +60,15 @@ SELECT * FROM sample WHERE data = 'B'; The Gravitino Flink connector transforms the following properties in a catalog to Flink connector configuration. -| Gravitino catalog property name | Flink Iceberg connector configuration | Description | Since Version | -|---------------------------------|---------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| -| `catalog-backend` | `catalog-type` | Catalog backend type, currently, only `Hive` Catalog is supported, `JDBC` and `Rest` in Continuous Validation | 0.8.0-incubating | -| `uri` | `uri` | Catalog backend URI | 0.8.0-incubating | -| `warehouse` | `warehouse` | Catalog backend warehouse | 0.8.0-incubating | -| `io-impl` | `io-impl` | The IO implementation for `FileIO` in Iceberg. | 0.8.0-incubating | -| `oss-endpoint` | `oss.endpoint` | The endpoint of Aliyun OSS service. | 0.8.0-incubating | -| `oss-access-key-id` | `client.access-key-id` | The static access key ID used to access OSS data. | 0.8.0-incubating | -| `oss-secret-access-key` | `client.access-key-secret` | The static secret access key used to access OSS data. | 0.8.0-incubating | +| Gravitino catalog property name | Flink Iceberg connector configuration | Description | Since Version | +|---------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------|------------------| +| `catalog-backend` | `catalog-type` | Catalog backend type, currently, only `Hive` Catalog is supported, `JDBC` and `Rest` in Continuous Validation | 0.8.0-incubating | +| `uri` | `uri` | Catalog backend URI | 0.8.0-incubating | +| `warehouse` | `warehouse` | Catalog backend warehouse | 0.8.0-incubating | +| `io-impl` | `io-impl` | The IO implementation for `FileIO` in Iceberg. | 0.8.0-incubating | +| `oss-endpoint` | `oss.endpoint` | The endpoint of Aliyun OSS service. | 0.8.0-incubating | +| `oss-access-key-id` | `client.access-key-id` | The static access key ID used to access OSS data. | 0.8.0-incubating | +| `oss-secret-access-key` | `client.access-key-secret` | The static secret access key used to access OSS data. | 0.8.0-incubating | Gravitino catalog property names with the prefix `flink.bypass.` are passed to Flink iceberg connector. For example, using `flink.bypass.clients` to pass the `clients` to the Flink iceberg connector. diff --git a/docs/flink-connector/flink-catalog-paimon.md b/docs/flink-connector/flink-catalog-paimon.md index 87b3451a8a0..e994233b3b4 100644 --- a/docs/flink-connector/flink-catalog-paimon.md +++ b/docs/flink-connector/flink-catalog-paimon.md @@ -6,6 +6,7 @@ license: "This software is licensed under the Apache License version 2." --- This document provides a comprehensive guide on configuring and using Apache Gravitino Flink connector to access the Paimon catalog managed by the Gravitino server. + ## Capabilities ### Supported Paimon Table Types @@ -32,7 +33,7 @@ Supports most DDL and DML operations in Flink SQL, except such operations: * Paimon 0.8 -Higher version like 0.9 or above may also supported but have not been tested fully. +Higher version like 0.9 or above may also support but have not been tested fully. ## Getting Started @@ -40,19 +41,18 @@ Higher version like 0.9 or above may also supported but have not been tested ful Place the following JAR files in the lib directory of your Flink installation: -* paimon-flink-1.18-0.8.2.jar - -* gravitino-flink-connector-runtime-\${flinkMajorVersion}_$scalaVersion.jar +- `paimon-flink-1.18-${paimon-version}.jar` +- `gravitino-flink-connector-runtime-1.18_2.12-${gravitino-version}.jar` ### SQL Example ```sql -- Suppose paimon_catalog is the Paimon catalog name managed by Gravitino -use catalog paimon_catalog; +USE CATALOG paimon_catalog; -- Execute statement succeed. -show databases; +SHOW DATABASES; -- +---------------------+ -- | database name | -- +---------------------+ @@ -66,12 +66,12 @@ SET 'execution.runtime-mode' = 'batch'; SET 'sql-client.execution.result-mode' = 'tableau'; -- [INFO] Execute statement succeed. -CREATE TABLE paimon_tabla_a ( +CREATE TABLE paimon_table_a ( aa BIGINT, bb BIGINT ); -show tables; +SHOW TABLES; -- +----------------+ -- | table name | -- +----------------+ @@ -79,15 +79,15 @@ show tables; -- +----------------+ -select * from paimon_table_a; +SELECT * FROM paimon_table_a; -- Empty set -insert into paimon_table_a(aa,bb) values(1,2); +INSERT INTO paimon_table_a(aa,bb) VALUES(1,2); -- [INFO] Submitting SQL update statement to the cluster... -- [INFO] SQL update statement has been successfully submitted to the cluster: -- Job ID: 74c0c678124f7b452daf08c399d0fee2 -select * from paimon_table_a; +SELECT * FROM paimon_table_a; -- +----+----+ -- | aa | bb | -- +----+----+ @@ -100,9 +100,9 @@ select * from paimon_table_a; Gravitino Flink connector will transform below property names which are defined in catalog properties to Flink Paimon connector configuration. -| Gravitino catalog property name | Flink Paimon connector configuration | Description | Since Version | -|---------------------------------|----------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| -| `catalog-backend` | `metastore` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem`. | 0.8.0-incubating | -| `warehouse` | `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-paimon/` for local fs, `hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or `oss://{bucket-name}/path` for Aliyun OSS | 0.8.0-incubating | +| Gravitino catalog property name | Flink Paimon connector configuration | Description | Since Version | +|---------------------------------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| +| `catalog-backend` | `metastore` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem`. | 0.8.0-incubating | +| `warehouse` | `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-paimon/` for local fs, `hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or `oss://{bucket-name}/path` for Aliyun OSS | 0.8.0-incubating | Gravitino catalog property names with the prefix `flink.bypass.` are passed to Flink Paimon connector. For example, using `flink.bypass.clients` to pass the `clients` to the Flink Paimon connector. diff --git a/docs/flink-connector/flink-connector.md b/docs/flink-connector/flink-connector.md index 3b43f5c4985..e6109bb378b 100644 --- a/docs/flink-connector/flink-connector.md +++ b/docs/flink-connector/flink-connector.md @@ -13,6 +13,8 @@ This capability allows users to perform federation queries, accessing data from ## Capabilities 1. Supports [Hive catalog](flink-catalog-hive.md) +1. Supports [Iceberg catalog](flink-catalog-iceberg.md) +1. Supports [Paimon catalog](flink-catalog-paimon.md) 2. Supports most DDL and DML SQLs. ## Requirement diff --git a/docs/manage-tags-in-gravitino.md b/docs/manage-tags-in-gravitino.md index 4163ca89d2c..04e5b79ec87 100644 --- a/docs/manage-tags-in-gravitino.md +++ b/docs/manage-tags-in-gravitino.md @@ -23,12 +23,12 @@ the future versions. :::info 1. Metadata objects are objects that are managed in Gravitino, such as `CATALOG`, `SCHEMA`, `TABLE`, - `COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, etc. A metadata object is combined by a `type` and a + `COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, `MODEL`, etc. A metadata object is combined by a `type` and a comma-separated `name`. For example, a `CATAGLOG` object has a name "catalog1" with type "CATALOG", a `SCHEMA` object has a name "catalog1.schema1" with type "SCHEMA", a `TABLE` object has a name "catalog1.schema1.table1" with type "TABLE", a `COLUMN` object has a name "catalog1.schema1.table1.column1" with type "COLUMN". -2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, and `COLUMN` objects can be tagged. +2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged. 3. Tags in Gravitino is inheritable, so listing tags of a metadata object will also list the tags of its parent metadata objects. For example, listing tags of a `Table` will also list the tags of its parent `Schema` and `Catalog`. @@ -204,8 +204,8 @@ client.deleteTag("tag2"); ## Tag associations -Gravitino allows you to associate and disassociate tags with metadata objects. Currently, only -`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC` objects can be tagged. +Gravitino allows you to associate and disassociate tags with metadata objects. Currently, +`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged. ### Associate and disassociate tags with a metadata object diff --git a/docs/open-api/catalogs.yaml b/docs/open-api/catalogs.yaml index 9e4efdaf588..1a9e9f8424d 100644 --- a/docs/open-api/catalogs.yaml +++ b/docs/open-api/catalogs.yaml @@ -355,7 +355,8 @@ components: enum: - relational - fileset - - stream + - messaging + - model provider: type: string description: The provider of the catalog diff --git a/docs/open-api/tags.yaml b/docs/open-api/tags.yaml index a3be5230b94..a8b3ac053c2 100644 --- a/docs/open-api/tags.yaml +++ b/docs/open-api/tags.yaml @@ -394,8 +394,8 @@ components: - "TABLE" - "FILESET" - "TOPIC" - - "ROLE" - - "METALAKE" + - "MODEL" + - "COLUMN" requests: diff --git a/docs/publish-docker-images.md b/docs/publish-docker-images.md index 953d3120662..d4590205ce6 100644 --- a/docs/publish-docker-images.md +++ b/docs/publish-docker-images.md @@ -30,9 +30,10 @@ You can use GitHub actions to publish Docker images to the Docker Hub repository 3. `apache/gravitino:0.1.0` if this is a gravitino server image. 4. `apache/gravitino-iceberg-rest:0.1.0` if this is an iceberg-rest server image. 6. You must enter the correct `docker user name`and `publish docker token` before you can execute run `Publish Docker Image` workflow. -7. Wait for the workflow to complete. You can see a new Docker image shown in the [Apache Docker Hub](https://hub.docker.com/u/apache) repository. +7. If you want to update the latest tag, select the box for `Whether to update the latest tag`. +8. Wait for the workflow to complete. You can see a new Docker image shown in the [Apache Docker Hub](https://hub.docker.com/u/apache) repository. -![Publish Docker image](assets/publish-docker-image.jpg) +![Publish Docker image](assets/publish-docker-image.png) ## More details of Apache Gravitino Docker images diff --git a/docs/spark-connector/spark-catalog-jdbc.md b/docs/spark-connector/spark-catalog-jdbc.md new file mode 100644 index 00000000000..7805d80266f --- /dev/null +++ b/docs/spark-connector/spark-catalog-jdbc.md @@ -0,0 +1,72 @@ +--- +title: "Spark connector JDBC catalog" +slug: /spark-connector/spark-catalog-jdbc +keyword: spark connector jdbc catalog +license: "This software is licensed under the Apache License version 2." +--- + +The Apache Gravitino Spark connector offers the capability to read JDBC tables, with the metadata managed by the Gravitino server. To enable the use of the JDBC catalog within the Spark connector, you must download the jdbc driver jar which you used to Spark classpath. + +## Capabilities + +Supports MySQL and PostgreSQL. For OceanBase which is compatible with Mysql Dialects could use Mysql driver and Mysql Dialects as a trackoff way. But for Doris which do not support MySQL Dialects, are not currently supported. + +#### Support DML and DDL operations: + +- `CREATE TABLE` +- `DROP TABLE` +- `ALTER TABLE` +- `SELECT` +- `INSERT` + + :::info + JDBCTable does not support distributed transaction. When writing data to RDBMS, each task is an independent transaction. If some tasks of spark succeed and some tasks fail, dirty data is generated. + ::: + +#### Not supported operations: + +- `UPDATE` +- `DELETE` +- `TRUNCATE` + +## SQL example + +```sql +-- Suppose mysql_a is the mysql catalog name managed by Gravitino +USE mysql_a; + +CREATE DATABASE IF NOT EXISTS mydatabase; +USE mydatabase; + +CREATE TABLE IF NOT EXISTS employee ( + id bigint, + name string, + department string, + hire_date timestamp +) +DESC TABLE EXTENDED employee; + +INSERT INTO employee +VALUES +(1, 'Alice', 'Engineering', TIMESTAMP '2021-01-01 09:00:00'), +(2, 'Bob', 'Marketing', TIMESTAMP '2021-02-01 10:30:00'), +(3, 'Charlie', 'Sales', TIMESTAMP '2021-03-01 08:45:00'); + +SELECT * FROM employee WHERE date(hire_date) = '2021-01-01'; + + +``` + +## Catalog properties + +Gravitino spark connector will transform below property names which are defined in catalog properties to Spark JDBC connector configuration. + +| Gravitino catalog property name | Spark JDBC connector configuration | Description | Since Version | +|---------------------------------|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| +| `jdbc-url` | `url` | JDBC URL for connecting to the database. For example, jdbc:mysql://localhost:3306 | 0.3.0 | +| `jdbc-user` | `jdbc.user` | JDBC user name | 0.3.0 | +| `jdbc-password` | `jdbc.password` | JDBC password | 0.3.0 | +| `jdbc-driver` | `driver` | The driver of the JDBC connection. For example, com.mysql.jdbc.Driver or com.mysql.cj.jdbc.Driver | 0.3.0 | + +Gravitino catalog property names with the prefix `spark.bypass.` are passed to Spark JDBC connector. + diff --git a/docs/spark-connector/spark-catalog-paimon.md b/docs/spark-connector/spark-catalog-paimon.md new file mode 100644 index 00000000000..2cae203e5db --- /dev/null +++ b/docs/spark-connector/spark-catalog-paimon.md @@ -0,0 +1,87 @@ +--- +title: "Spark connector Paimon catalog" +slug: /spark-connector/spark-catalog-paimon +keyword: spark connector paimon catalog +license: "This software is licensed under the Apache License version 2." +--- + +The Apache Gravitino Spark connector offers the capability to read and write Paimon tables, with the metadata managed by the Gravitino server. To enable the use of the Paimon catalog within the Spark connector now, you must set download [Paimon Spark runtime jar](https://paimon.apache.org/docs/0.8/spark/quick-start/#preparation) to Spark classpath. + +## Capabilities + +### Support DDL and DML operations: + +- `CREATE NAMESPACE` +- `DROP NAMESPACE` +- `LIST NAMESPACE` +- `LOAD NAMESPACE` + - It can not return any user-specified configs now, as we only support FilesystemCatalog in spark-connector now. +- `CREATE TABLE` + - Doesn't support distribution and sort orders. +- `DROP TABLE` +- `ALTER TABLE` +- `LIST TABLE` +- `DESRICE TABLE` +- `SELECT` +- `INSERT INTO & OVERWRITE` +- `Schema Evolution` +- `PARTITION MANAGEMENT`, such as `LIST PARTITIONS`, `ALTER TABLE ... DROP PARTITION ...` + +:::info +Only supports Paimon FilesystemCatalog on HDFS now. +::: + +#### Not supported operations: + +- `ALTER NAMESPACE` + - Paimon does not support alter namespace. +- Row Level operations, such as `MERGE INTO`, `DELETE`, `UPDATE`, `TRUNCATE` +- Metadata tables, such as `{paimon_catalog}.{paimon_database}.{paimon_table}$snapshots` +- Other Paimon extension SQLs, such as `Tag` +- Call Statements +- View +- Time Travel +- Hive and Jdbc backend, and Object Storage for FilesystemCatalog + +## SQL example + +```sql +-- Suppose paimon_catalog is the Paimon catalog name managed by Gravitino +USE paimon_catalog; + +CREATE DATABASE IF NOT EXISTS mydatabase; +USE mydatabase; + +CREATE TABLE IF NOT EXISTS employee ( + id bigint, + name string, + department string, + hire_date timestamp +) PARTITIONED BY (name); + +SHOW TABLES; +DESC TABLE EXTENDED employee; + +INSERT INTO employee +VALUES +(1, 'Alice', 'Engineering', TIMESTAMP '2021-01-01 09:00:00'), +(2, 'Bob', 'Marketing', TIMESTAMP '2021-02-01 10:30:00'), +(3, 'Charlie', 'Sales', TIMESTAMP '2021-03-01 08:45:00'); + +SELECT * FROM employee WHERE name = 'Alice'; + +SHOW PARTITIONS employee; +ALTER TABLE employee DROP PARTITION (`name`='Alice'); +``` + +## Catalog properties + +Gravitino spark connector will transform below property names which are defined in catalog properties to Spark Paimon connector configuration. + +| Gravitino catalog property name | Spark Paimon connector configuration | Description | Since Version | +|---------------------------------|--------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| +| `catalog-backend` | `metastore` | Catalog backend type | 0.8.0-incubating | +| `uri` | `uri` | Catalog backend uri | 0.8.0-incubating | +| `warehouse` | `warehouse` | Catalog backend warehouse | 0.8.0-incubating | + +Gravitino catalog property names with the prefix `spark.bypass.` are passed to Spark Paimon connector. For example, using `spark.bypass.client-pool-size` to pass the `client-pool-size` to the Spark Paimon connector. diff --git a/docs/spark-connector/spark-connector.md b/docs/spark-connector/spark-connector.md index 44179e05f55..a982313433a 100644 --- a/docs/spark-connector/spark-connector.md +++ b/docs/spark-connector/spark-connector.md @@ -11,7 +11,7 @@ The Apache Gravitino Spark connector leverages the Spark DataSourceV2 interface ## Capabilities -1. Supports [Hive catalog](spark-catalog-hive.md) and [Iceberg catalog](spark-catalog-iceberg.md). +1. Supports [Hive catalog](spark-catalog-hive.md), [Iceberg catalog](spark-catalog-iceberg.md), [Paimon catalog](spark-catalog-paimon.md) and [Jdbc catalog](spark-catalog-jdbc.md). 2. Supports federation query. 3. Supports most DDL and DML SQLs. diff --git a/flink-connector/flink-runtime/build.gradle.kts b/flink-connector/flink-runtime/build.gradle.kts index 1a71646444e..0ac393dedf9 100644 --- a/flink-connector/flink-runtime/build.gradle.kts +++ b/flink-connector/flink-runtime/build.gradle.kts @@ -41,6 +41,14 @@ val scalaVersion: String = "2.12" val artifactName = "gravitino-${project.name}_$scalaVersion" val baseName = "${rootProject.name}-flink-connector-runtime-${flinkMajorVersion}_$scalaVersion" +configurations.all { + resolutionStrategy.eachDependency { + if (requested.group == "org.apache.logging.log4j") { + throw GradleException("Forbidden dependency 'org.apache.logging.log4j' found!") + } + } +} + dependencies { implementation(project(":clients:client-java-runtime", configuration = "shadow")) implementation(project(":flink-connector:flink")) diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 4c9bd036ae9..6cbfbfa53b4 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -41,8 +41,12 @@ val scalaVersion: String = "2.12" val artifactName = "${rootProject.name}-flink-${flinkMajorVersion}_$scalaVersion" dependencies { - implementation(project(":core")) - implementation(project(":catalogs:catalog-common")) + implementation(project(":core")) { + exclude("org.apache.logging.log4j") + } + implementation(project(":catalogs:catalog-common")) { + exclude("org.apache.logging.log4j") + } implementation(libs.guava) compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java index c9fbb8a491c..15d1a12fa3b 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java @@ -19,8 +19,10 @@ package org.apache.gravitino.flink.connector; +import com.google.common.collect.Maps; import java.util.Map; import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.CommonCatalogOptions; /** * PropertiesConverter is used to convert properties between Flink properties and Apache Gravitino @@ -32,25 +34,82 @@ public interface PropertiesConverter { /** * Converts properties from application provided properties and Flink connector properties to - * Gravitino properties. + * Gravitino properties.This method processes the Flink configuration and transforms it into a + * format suitable for the Gravitino catalog. * - * @param flinkConf The configuration provided by Flink. - * @return properties for the Gravitino catalog. + * @param flinkConf The Flink configuration containing connector properties. This includes both + * Flink-specific properties and any user-provided properties. + * @return A map of properties converted for use in the Gravitino catalog. The returned map + * includes both directly transformed properties and bypass properties prefixed with {@link + * #FLINK_PROPERTY_PREFIX}. */ default Map toGravitinoCatalogProperties(Configuration flinkConf) { - return flinkConf.toMap(); + Map gravitinoProperties = Maps.newHashMap(); + for (Map.Entry entry : flinkConf.toMap().entrySet()) { + String gravitinoKey = transformPropertyToGravitinoCatalog(entry.getKey()); + if (gravitinoKey != null) { + gravitinoProperties.put(gravitinoKey, entry.getValue()); + } else if (!entry.getKey().startsWith(FLINK_PROPERTY_PREFIX)) { + gravitinoProperties.put(FLINK_PROPERTY_PREFIX + entry.getKey(), entry.getValue()); + } else { + gravitinoProperties.put(entry.getKey(), entry.getValue()); + } + } + return gravitinoProperties; } /** - * Converts properties from Gravitino properties to Flink connector properties. + * Converts properties from Gravitino catalog properties to Flink connector properties. This + * method processes the Gravitino properties and transforms them into a format suitable for the + * Flink connector. * - * @param gravitinoProperties The properties provided by Gravitino. - * @return properties for the Flink connector. + * @param gravitinoProperties The properties provided by the Gravitino catalog. This includes both + * Gravitino-specific properties and any bypass properties prefixed with {@link + * #FLINK_PROPERTY_PREFIX}. + * @return A map of properties converted for use in the Flink connector. The returned map includes + * both transformed properties and the Flink catalog type. */ default Map toFlinkCatalogProperties(Map gravitinoProperties) { - return gravitinoProperties; + Map allProperties = Maps.newHashMap(); + gravitinoProperties.forEach( + (key, value) -> { + String flinkConfigKey = key; + if (key.startsWith(PropertiesConverter.FLINK_PROPERTY_PREFIX)) { + flinkConfigKey = key.substring(PropertiesConverter.FLINK_PROPERTY_PREFIX.length()); + allProperties.put(flinkConfigKey, value); + } else { + String convertedKey = transformPropertyToFlinkCatalog(flinkConfigKey); + if (convertedKey != null) { + allProperties.put(convertedKey, value); + } + } + }); + allProperties.put(CommonCatalogOptions.CATALOG_TYPE.key(), getFlinkCatalogType()); + return allProperties; } + /** + * Transforms a Flink configuration key to a corresponding Gravitino catalog property key. This + * method is used to map Flink-specific configuration keys to Gravitino catalog properties. + * + * @param configKey The Flink configuration key to be transformed. + * @return The corresponding Gravitino catalog property key, or {@code null} if no transformation + * is needed. + */ + String transformPropertyToGravitinoCatalog(String configKey); + + /** + * Transforms a specific configuration key from Gravitino catalog properties to Flink connector + * properties. This method is used to convert a property key that is specific to Gravitino into a + * format that can be understood by the Flink connector. + * + * @param configKey The configuration key from Gravitino catalog properties to be transformed. + * @return The transformed configuration key that is compatible with the Flink connector. + * @throws IllegalArgumentException If the provided configuration key cannot be transformed or is + * invalid. + */ + String transformPropertyToFlinkCatalog(String configKey); + /** * Converts properties from Flink connector schema properties to Gravitino schema properties. * @@ -90,4 +149,12 @@ default Map toFlinkTableProperties(Map gravitino default Map toGravitinoTableProperties(Map flinkProperties) { return flinkProperties; } + + /** + * Retrieves the Flink catalog type associated with this converter. This method is used to + * determine the type of Flink catalog that this converter is designed for. + * + * @return The Flink catalog type. + */ + String getFlinkCatalogType(); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java index 18095867498..20a3e8cf62d 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java @@ -20,11 +20,8 @@ package org.apache.gravitino.flink.connector.hive; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import java.util.Map; import java.util.stream.Collectors; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.hadoop.hive.conf.HiveConf; @@ -34,46 +31,19 @@ public class HivePropertiesConverter implements PropertiesConverter { private HivePropertiesConverter() {} public static final HivePropertiesConverter INSTANCE = new HivePropertiesConverter(); - private static final Map HIVE_CATALOG_CONFIG_TO_GRAVITINO = ImmutableMap.of(HiveConf.ConfVars.METASTOREURIS.varname, HiveConstants.METASTORE_URIS); private static final Map GRAVITINO_CONFIG_TO_HIVE = ImmutableMap.of(HiveConstants.METASTORE_URIS, HiveConf.ConfVars.METASTOREURIS.varname); @Override - public Map toGravitinoCatalogProperties(Configuration flinkConf) { - Map gravitinoProperties = Maps.newHashMap(); - - for (Map.Entry entry : flinkConf.toMap().entrySet()) { - String gravitinoKey = HIVE_CATALOG_CONFIG_TO_GRAVITINO.get(entry.getKey()); - if (gravitinoKey != null) { - gravitinoProperties.put(gravitinoKey, entry.getValue()); - } else if (!entry.getKey().startsWith(FLINK_PROPERTY_PREFIX)) { - gravitinoProperties.put(FLINK_PROPERTY_PREFIX + entry.getKey(), entry.getValue()); - } else { - gravitinoProperties.put(entry.getKey(), entry.getValue()); - } - } - - return gravitinoProperties; + public String transformPropertyToGravitinoCatalog(String configKey) { + return HIVE_CATALOG_CONFIG_TO_GRAVITINO.get(configKey); } @Override - public Map toFlinkCatalogProperties(Map gravitinoProperties) { - Map flinkCatalogProperties = Maps.newHashMap(); - flinkCatalogProperties.put( - CommonCatalogOptions.CATALOG_TYPE.key(), GravitinoHiveCatalogFactoryOptions.IDENTIFIER); - - gravitinoProperties.forEach( - (key, value) -> { - String flinkConfigKey = key; - if (key.startsWith(PropertiesConverter.FLINK_PROPERTY_PREFIX)) { - flinkConfigKey = key.substring(PropertiesConverter.FLINK_PROPERTY_PREFIX.length()); - } - flinkCatalogProperties.put( - GRAVITINO_CONFIG_TO_HIVE.getOrDefault(flinkConfigKey, flinkConfigKey), value); - }); - return flinkCatalogProperties; + public String transformPropertyToFlinkCatalog(String configKey) { + return GRAVITINO_CONFIG_TO_HIVE.get(configKey); } @Override @@ -95,4 +65,9 @@ public Map toFlinkTableProperties(Map gravitinoP properties.put("connector", "hive"); return properties; } + + @Override + public String getFlinkCatalogType() { + return GravitinoHiveCatalogFactoryOptions.IDENTIFIER; + } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/IcebergPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/IcebergPropertiesConverter.java index 7684d3eadbb..1d80e27ea59 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/IcebergPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/IcebergPropertiesConverter.java @@ -19,11 +19,9 @@ package org.apache.gravitino.flink.connector.iceberg; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.Map; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils; import org.apache.gravitino.flink.connector.PropertiesConverter; @@ -38,38 +36,21 @@ private IcebergPropertiesConverter() {} IcebergConstants.CATALOG_BACKEND, IcebergPropertiesConstants.ICEBERG_CATALOG_TYPE); @Override - public Map toFlinkCatalogProperties(Map gravitinoProperties) { - Preconditions.checkArgument( - gravitinoProperties != null, "Iceberg Catalog properties should not be null."); + public String transformPropertyToGravitinoCatalog(String configKey) { + return IcebergPropertiesUtils.ICEBERG_CATALOG_CONFIG_TO_GRAVITINO.get(configKey); + } - Map all = new HashMap<>(); - if (gravitinoProperties != null) { - gravitinoProperties.forEach( - (k, v) -> { - if (k.startsWith(FLINK_PROPERTY_PREFIX)) { - String newKey = k.substring(FLINK_PROPERTY_PREFIX.length()); - all.put(newKey, v); - } - }); - } - Map transformedProperties = - IcebergPropertiesUtils.toIcebergCatalogProperties(gravitinoProperties); + @Override + public String transformPropertyToFlinkCatalog(String configKey) { - if (transformedProperties != null) { - all.putAll(transformedProperties); + String icebergConfigKey = null; + if (IcebergPropertiesUtils.GRAVITINO_CONFIG_TO_ICEBERG.containsKey(configKey)) { + icebergConfigKey = IcebergPropertiesUtils.GRAVITINO_CONFIG_TO_ICEBERG.get(configKey); } - all.put( - CommonCatalogOptions.CATALOG_TYPE.key(), GravitinoIcebergCatalogFactoryOptions.IDENTIFIER); - // Map "catalog-backend" to "catalog-type". - // TODO If catalog backend is CUSTOM, we need special compatibility logic. - GRAVITINO_CONFIG_TO_FLINK_ICEBERG.forEach( - (key, value) -> { - if (all.containsKey(key)) { - String config = all.remove(key); - all.put(value, config); - } - }); - return all; + if (GRAVITINO_CONFIG_TO_FLINK_ICEBERG.containsKey(configKey)) { + icebergConfigKey = GRAVITINO_CONFIG_TO_FLINK_ICEBERG.get(configKey); + } + return icebergConfigKey; } @Override @@ -78,7 +59,7 @@ public Map toGravitinoTableProperties(Map proper } @Override - public Map toFlinkTableProperties(Map properties) { - return new HashMap<>(properties); + public String getFlinkCatalogType() { + return GravitinoIcebergCatalogFactoryOptions.IDENTIFIER; } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java index c22e00fa122..06107b86248 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java @@ -24,11 +24,13 @@ import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.CatalogException; import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.apache.flink.table.factories.CatalogFactory; import org.apache.flink.table.factories.Factory; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.flink.connector.PartitionConverter; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalog; +import org.apache.paimon.flink.FlinkCatalogFactory; import org.apache.paimon.flink.FlinkTableFactory; /** @@ -40,12 +42,13 @@ public class GravitinoPaimonCatalog extends BaseCatalog { private final AbstractCatalog paimonCatalog; protected GravitinoPaimonCatalog( - String catalogName, - AbstractCatalog paimonCatalog, + CatalogFactory.Context context, + String defaultDatabase, PropertiesConverter propertiesConverter, PartitionConverter partitionConverter) { - super(catalogName, paimonCatalog.getDefaultDatabase(), propertiesConverter, partitionConverter); - this.paimonCatalog = paimonCatalog; + super(context.getName(), defaultDatabase, propertiesConverter, partitionConverter); + FlinkCatalogFactory flinkCatalogFactory = new FlinkCatalogFactory(); + this.paimonCatalog = flinkCatalogFactory.createCatalog(context); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java index 52489fc667f..8732ade23ed 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java @@ -23,12 +23,12 @@ import java.util.Set; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.factories.FactoryUtil; import org.apache.gravitino.flink.connector.DefaultPartitionConverter; import org.apache.gravitino.flink.connector.PartitionConverter; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; -import org.apache.paimon.flink.FlinkCatalog; -import org.apache.paimon.flink.FlinkCatalogFactory; +import org.apache.gravitino.flink.connector.utils.FactoryUtils; /** * Factory for creating instances of {@link GravitinoPaimonCatalog}. It will be created by SPI @@ -38,9 +38,12 @@ public class GravitinoPaimonCatalogFactory implements BaseCatalogFactory { @Override public Catalog createCatalog(Context context) { - FlinkCatalog catalog = new FlinkCatalogFactory().createCatalog(context); + final FactoryUtil.CatalogFactoryHelper helper = + FactoryUtils.createCatalogFactoryHelper(this, context); + String defaultDatabase = + helper.getOptions().get(GravitinoPaimonCatalogFactoryOptions.DEFAULT_DATABASE); return new GravitinoPaimonCatalog( - context.getName(), catalog, propertiesConverter(), partitionConverter()); + context, defaultDatabase, propertiesConverter(), partitionConverter()); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java index dd78f96d24b..a4180b9eb40 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java @@ -19,8 +19,17 @@ package org.apache.gravitino.flink.connector.paimon; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.paimon.flink.FlinkCatalogOptions; + public class GravitinoPaimonCatalogFactoryOptions { /** Identifier for the {@link GravitinoPaimonCatalog}. */ public static final String IDENTIFIER = "gravitino-paimon"; + + public static final ConfigOption DEFAULT_DATABASE = + ConfigOptions.key(FlinkCatalogOptions.DEFAULT_DATABASE.key()) + .stringType() + .defaultValue(FlinkCatalogOptions.DEFAULT_DATABASE.defaultValue()); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java index 58613bee37d..99e402bcb88 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java @@ -19,15 +19,9 @@ package org.apache.gravitino.flink.connector.paimon; -import com.google.common.collect.Maps; -import java.util.HashMap; -import java.util.Map; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonPropertiesUtils; import org.apache.gravitino.flink.connector.PropertiesConverter; -import org.apache.paimon.catalog.FileSystemCatalogFactory; public class PaimonPropertiesConverter implements PropertiesConverter { @@ -36,45 +30,23 @@ public class PaimonPropertiesConverter implements PropertiesConverter { private PaimonPropertiesConverter() {} @Override - public Map toGravitinoCatalogProperties(Configuration flinkConf) { - Map gravitinoProperties = Maps.newHashMap(); - Map flinkConfMap = flinkConf.toMap(); - for (Map.Entry entry : flinkConfMap.entrySet()) { - String gravitinoKey = - PaimonPropertiesUtils.PAIMON_CATALOG_CONFIG_TO_GRAVITINO.get(entry.getKey()); - if (gravitinoKey != null) { - gravitinoProperties.put(gravitinoKey, entry.getValue()); - } else if (!entry.getKey().startsWith(FLINK_PROPERTY_PREFIX)) { - gravitinoProperties.put(FLINK_PROPERTY_PREFIX + entry.getKey(), entry.getValue()); - } else { - gravitinoProperties.put(entry.getKey(), entry.getValue()); - } + public String transformPropertyToGravitinoCatalog(String configKey) { + if (configKey.equalsIgnoreCase(PaimonConstants.METASTORE)) { + return PaimonConstants.CATALOG_BACKEND; } - gravitinoProperties.put( - PaimonConstants.CATALOG_BACKEND, - flinkConfMap.getOrDefault(PaimonConstants.METASTORE, FileSystemCatalogFactory.IDENTIFIER)); - return gravitinoProperties; + return PaimonPropertiesUtils.PAIMON_CATALOG_CONFIG_TO_GRAVITINO.get(configKey); } @Override - public Map toFlinkCatalogProperties(Map gravitinoProperties) { - Map all = new HashMap<>(); - gravitinoProperties.forEach( - (key, value) -> { - String flinkConfigKey = key; - if (key.startsWith(PropertiesConverter.FLINK_PROPERTY_PREFIX)) { - flinkConfigKey = key.substring(PropertiesConverter.FLINK_PROPERTY_PREFIX.length()); - } - all.put(flinkConfigKey, value); - }); - Map paimonCatalogProperties = - PaimonPropertiesUtils.toPaimonCatalogProperties(all); - paimonCatalogProperties.put( - PaimonConstants.METASTORE, - paimonCatalogProperties.getOrDefault( - PaimonConstants.CATALOG_BACKEND, FileSystemCatalogFactory.IDENTIFIER)); - paimonCatalogProperties.put( - CommonCatalogOptions.CATALOG_TYPE.key(), GravitinoPaimonCatalogFactoryOptions.IDENTIFIER); - return paimonCatalogProperties; + public String transformPropertyToFlinkCatalog(String configKey) { + if (configKey.equals(PaimonConstants.CATALOG_BACKEND)) { + return PaimonConstants.METASTORE; + } + return PaimonPropertiesUtils.GRAVITINO_CONFIG_TO_PAIMON.get(configKey); + } + + @Override + public String getFlinkCatalogType() { + return GravitinoPaimonCatalogFactoryOptions.IDENTIFIER; } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/iceberg/TestIcebergPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/iceberg/TestIcebergPropertiesConverter.java index d6de522f396..8287eebf25e 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/iceberg/TestIcebergPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/iceberg/TestIcebergPropertiesConverter.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.apache.flink.configuration.Configuration; import org.apache.flink.table.catalog.CommonCatalogOptions; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -31,16 +32,37 @@ public class TestIcebergPropertiesConverter { @Test void testCatalogPropertiesWithHiveBackend() { Map properties = - CONVERTER.toFlinkCatalogProperties( - ImmutableMap.of( - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_HIVE, - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_URI, - "hive-uri", - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_WAREHOUSE, - "hive-warehouse", - "key1", - "value1")); + CONVERTER.toGravitinoCatalogProperties( + Configuration.fromMap( + ImmutableMap.of( + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_HIVE, + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_URI, + "hive-uri", + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_WAREHOUSE, + "hive-warehouse", + "key1", + "value1", + "flink.bypass.key2", + "value2"))); + + Map actual = + ImmutableMap.of( + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_HIVE, + IcebergPropertiesConstants.ICEBERG_CATALOG_URI, + "hive-uri", + IcebergPropertiesConstants.ICEBERG_CATALOG_WAREHOUSE, + "hive-warehouse", + "flink.bypass.key1", // Automatically add 'flink.bypass.' + "value1", + "flink.bypass.key2", + "value2"); + + Assertions.assertEquals(actual, properties); + + Map toFlinkProperties = CONVERTER.toFlinkCatalogProperties(actual); + Assertions.assertEquals( ImmutableMap.of( CommonCatalogOptions.CATALOG_TYPE.key(), @@ -50,23 +72,48 @@ void testCatalogPropertiesWithHiveBackend() { IcebergPropertiesConstants.ICEBERG_CATALOG_URI, "hive-uri", IcebergPropertiesConstants.ICEBERG_CATALOG_WAREHOUSE, - "hive-warehouse"), - properties); + "hive-warehouse", + "key1", // When returning to Flink, prefix 'flink.bypass.' Automatically removed. + "value1", + "key2", // When returning to Flink, prefix 'flink.bypass.' Automatically removed. + "value2"), + toFlinkProperties); } @Test void testCatalogPropertiesWithRestBackend() { Map properties = - CONVERTER.toFlinkCatalogProperties( - ImmutableMap.of( - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, - IcebergPropertiesConstants.ICEBERG_CATALOG_BACKEND_REST, - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_URI, - "rest-uri", - IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_WAREHOUSE, - "rest-warehouse", - "key1", - "value1")); + CONVERTER.toGravitinoCatalogProperties( + Configuration.fromMap( + ImmutableMap.of( + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, + IcebergPropertiesConstants.ICEBERG_CATALOG_BACKEND_REST, + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_URI, + "rest-uri", + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_WAREHOUSE, + "rest-warehouse", + "key1", + "value1", + "flink.bypass.key2", + "value2"))); + + Map actual = + ImmutableMap.of( + IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND, + IcebergPropertiesConstants.ICEBERG_CATALOG_BACKEND_REST, + IcebergPropertiesConstants.ICEBERG_CATALOG_URI, + "rest-uri", + IcebergPropertiesConstants.ICEBERG_CATALOG_WAREHOUSE, + "rest-warehouse", + "flink.bypass.key1", // Automatically add 'flink.bypass.' + "value1", + "flink.bypass.key2", + "value2"); + + Assertions.assertEquals(actual, properties); + + Map toFlinkProperties = CONVERTER.toFlinkCatalogProperties(actual); + Assertions.assertEquals( ImmutableMap.of( CommonCatalogOptions.CATALOG_TYPE.key(), @@ -76,7 +123,11 @@ void testCatalogPropertiesWithRestBackend() { IcebergPropertiesConstants.ICEBERG_CATALOG_URI, "rest-uri", IcebergPropertiesConstants.ICEBERG_CATALOG_WAREHOUSE, - "rest-warehouse"), - properties); + "rest-warehouse", + "key1", // When returning to Flink, prefix 'flink.bypass.' Automatically removed. + "value1", + "key2", // When returning to Flink, prefix 'flink.bypass.' Automatically removed. + "value2"), + toFlinkProperties); } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java index 0834def90b7..cedf0e8d591 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java @@ -157,10 +157,6 @@ public void testCreateGravitinoIcebergUsingSQL() { Map properties = gravitinoCatalog.properties(); Assertions.assertEquals(hiveMetastoreUri, properties.get(IcebergConstants.URI)); - Assertions.assertEquals( - GravitinoIcebergCatalogFactoryOptions.IDENTIFIER, - properties.get(CommonCatalogOptions.CATALOG_TYPE.key())); - // Get the created catalog. Optional catalog = tableEnv.getCatalog(catalogName); Assertions.assertTrue(catalog.isPresent()); diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java index a03b4a198e1..66458ba8e74 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java @@ -98,7 +98,7 @@ public void testCreateGravitinoPaimonCatalogUsingSQL() { "create catalog %s with (" + "'type'='gravitino-paimon', " + "'warehouse'='%s'," - + "'catalog.backend'='filesystem'" + + "'metastore'='filesystem'" + ")", catalogName, warehouse)); String[] catalogs = tableEnv.listCatalogs(); diff --git a/server-common/src/main/java/org/apache/gravitino/server/authentication/KerberosServerUtils.java b/server-common/src/main/java/org/apache/gravitino/server/authentication/KerberosServerUtils.java index 3813b26bbfb..4db1a53bc28 100644 --- a/server-common/src/main/java/org/apache/gravitino/server/authentication/KerberosServerUtils.java +++ b/server-common/src/main/java/org/apache/gravitino/server/authentication/KerberosServerUtils.java @@ -16,12 +16,11 @@ import java.io.File; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.nio.charset.IllegalCharsetNameException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; @@ -353,11 +352,8 @@ DER get(int... tags) { } String getAsString() { - try { - return new String(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining(), "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalCharsetNameException("UTF-8"); // won't happen. - } + return new String( + bb.array(), bb.arrayOffset() + bb.position(), bb.remaining(), StandardCharsets.UTF_8); } @Override diff --git a/spark-connector/spark-common/build.gradle.kts b/spark-connector/spark-common/build.gradle.kts index 06e0077d21e..dc6bbd258e7 100644 --- a/spark-connector/spark-common/build.gradle.kts +++ b/spark-connector/spark-common/build.gradle.kts @@ -40,6 +40,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat dependencies { implementation(project(":catalogs:catalog-common")) implementation(libs.guava) + implementation(libs.caffeine) compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java index f04193f33a3..a484972656a 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java @@ -18,13 +18,12 @@ */ package org.apache.gravitino.spark.connector.catalog; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.ExecutionException; import org.apache.gravitino.Catalog; import org.apache.gravitino.client.GravitinoClient; import org.slf4j.Logger; @@ -42,7 +41,7 @@ public class GravitinoCatalogManager { private GravitinoCatalogManager(Supplier clientBuilder) { this.gravitinoClient = clientBuilder.get(); // Will not evict catalog by default - this.gravitinoCatalogs = CacheBuilder.newBuilder().build(); + this.gravitinoCatalogs = Caffeine.newBuilder().build(); } public static GravitinoCatalogManager create(Supplier clientBuilder) { @@ -69,8 +68,8 @@ public void close() { public Catalog getGravitinoCatalogInfo(String name) { try { - return gravitinoCatalogs.get(name, () -> loadCatalog(name)); - } catch (ExecutionException e) { + return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName)); + } catch (Exception e) { LOG.error(String.format("Load catalog %s failed", name), e); throw new RuntimeException(e); } diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java new file mode 100644 index 00000000000..3f36b9a2a84 --- /dev/null +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.gravitino.spark.connector.PropertiesConverter; +import org.apache.gravitino.spark.connector.SparkTransformConverter; +import org.apache.gravitino.spark.connector.SparkTypeConverter; +import org.apache.gravitino.spark.connector.catalog.BaseCatalog; +import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.SupportsNamespaces; +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.errors.QueryCompilationErrors; +import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTable; +import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +public class GravitinoJdbcCatalog extends BaseCatalog { + + @Override + protected TableCatalog createAndInitSparkCatalog( + String name, CaseInsensitiveStringMap options, Map properties) { + JDBCTableCatalog jdbcTableCatalog = new JDBCTableCatalog(); + Map all = + getPropertiesConverter().toSparkCatalogProperties(options, properties); + jdbcTableCatalog.initialize(name, new CaseInsensitiveStringMap(all)); + return jdbcTableCatalog; + } + + @Override + protected Table createSparkTable( + Identifier identifier, + org.apache.gravitino.rel.Table gravitinoTable, + Table sparkTable, + TableCatalog sparkCatalog, + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter, + SparkTypeConverter sparkTypeConverter) { + return new SparkJdbcTable( + identifier, + gravitinoTable, + (JDBCTable) sparkTable, + (JDBCTableCatalog) sparkCatalog, + propertiesConverter, + sparkTransformConverter, + sparkTypeConverter); + } + + @Override + protected PropertiesConverter getPropertiesConverter() { + return JdbcPropertiesConverter.getInstance(); + } + + @Override + protected SparkTransformConverter getSparkTransformConverter() { + return new SparkTransformConverter(false); + } + + @Override + protected SparkTypeConverter getSparkTypeConverter() { + return new SparkJdbcTypeConverter(); + } + + @Override + public void createNamespace(String[] namespace, Map metadata) + throws NamespaceAlreadyExistsException { + Map properties = Maps.newHashMap(); + if (!metadata.isEmpty()) { + metadata.forEach( + (k, v) -> { + switch (k) { + case SupportsNamespaces.PROP_COMMENT: + properties.put(k, v); + break; + case SupportsNamespaces.PROP_OWNER: + break; + case SupportsNamespaces.PROP_LOCATION: + throw new RuntimeException( + QueryCompilationErrors.cannotCreateJDBCNamespaceUsingProviderError()); + default: + throw new RuntimeException( + QueryCompilationErrors.cannotCreateJDBCNamespaceWithPropertyError(k)); + } + }); + } + super.createNamespace(namespace, properties); + } +} diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConstants.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConstants.java new file mode 100644 index 00000000000..f1cf50f81d3 --- /dev/null +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConstants.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +public class JdbcPropertiesConstants { + + public static final String GRAVITINO_JDBC_USER = "jdbc-user"; + public static final String GRAVITINO_JDBC_PASSWORD = "jdbc-password"; + public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver"; + public static final String GRAVITINO_JDBC_URL = "jdbc-url"; + + public static final String SPARK_JDBC_URL = "url"; + public static final String SPARK_JDBC_USER = "user"; + public static final String SPARK_JDBC_PASSWORD = "password"; + public static final String SPARK_JDBC_DRIVER = "driver"; +} diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java new file mode 100644 index 00000000000..7516646e343 --- /dev/null +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.spark.connector.PropertiesConverter; + +public class JdbcPropertiesConverter implements PropertiesConverter { + + public static class JdbcPropertiesConverterHolder { + private static final JdbcPropertiesConverter INSTANCE = new JdbcPropertiesConverter(); + } + + private JdbcPropertiesConverter() {} + + public static JdbcPropertiesConverter getInstance() { + return JdbcPropertiesConverterHolder.INSTANCE; + } + + private static final Map GRAVITINO_CONFIG_TO_JDBC = + ImmutableMap.of( + JdbcPropertiesConstants.GRAVITINO_JDBC_URL, + JdbcPropertiesConstants.SPARK_JDBC_URL, + JdbcPropertiesConstants.GRAVITINO_JDBC_USER, + JdbcPropertiesConstants.SPARK_JDBC_USER, + JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, + JdbcPropertiesConstants.SPARK_JDBC_PASSWORD, + JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER, + JdbcPropertiesConstants.SPARK_JDBC_DRIVER); + + @Override + public Map toSparkCatalogProperties(Map properties) { + Preconditions.checkArgument(properties != null, "Jdbc Catalog properties should not be null"); + HashMap jdbcProperties = new HashMap<>(); + properties.forEach( + (key, value) -> { + if (GRAVITINO_CONFIG_TO_JDBC.containsKey(key)) { + jdbcProperties.put(GRAVITINO_CONFIG_TO_JDBC.get(key), value); + } + }); + return jdbcProperties; + } + + @Override + public Map toGravitinoTableProperties(Map properties) { + return new HashMap<>(properties); + } + + @Override + public Map toSparkTableProperties(Map properties) { + return new HashMap<>(properties); + } +} diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTable.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTable.java new file mode 100644 index 00000000000..3de807c3685 --- /dev/null +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTable.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import java.util.Map; +import org.apache.gravitino.rel.Table; +import org.apache.gravitino.spark.connector.PropertiesConverter; +import org.apache.gravitino.spark.connector.SparkTransformConverter; +import org.apache.gravitino.spark.connector.SparkTypeConverter; +import org.apache.gravitino.spark.connector.utils.GravitinoTableInfoHelper; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTable; +import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog; +import org.apache.spark.sql.types.StructType; + +public class SparkJdbcTable extends JDBCTable { + + private GravitinoTableInfoHelper gravitinoTableInfoHelper; + + public SparkJdbcTable( + Identifier identifier, + Table gravitinoTable, + JDBCTable jdbcTable, + JDBCTableCatalog jdbcTableCatalog, + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter, + SparkTypeConverter sparkTypeConverter) { + super(identifier, jdbcTable.schema(), jdbcTable.jdbcOptions()); + this.gravitinoTableInfoHelper = + new GravitinoTableInfoHelper( + false, + identifier, + gravitinoTable, + propertiesConverter, + sparkTransformConverter, + sparkTypeConverter); + } + + @Override + public String name() { + return gravitinoTableInfoHelper.name(); + } + + @Override + @SuppressWarnings("deprecation") + public StructType schema() { + return gravitinoTableInfoHelper.schema(); + } + + @Override + public Map properties() { + return gravitinoTableInfoHelper.properties(); + } +} diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java new file mode 100644 index 00000000000..56e2734a7f4 --- /dev/null +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; +import org.apache.gravitino.spark.connector.SparkTypeConverter; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; + +public class SparkJdbcTypeConverter extends SparkTypeConverter { + + @Override + public DataType toSparkType(Type gravitinoType) { + // if spark version lower than 3.4.4, using VarCharType will throw an exception: Unsupported + // type varchar. + if (gravitinoType instanceof Types.VarCharType) { + return DataTypes.StringType; + } else { + return super.toSparkType(gravitinoType); + } + } +} diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java index 9392feac2f1..9d8594b9124 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java @@ -46,11 +46,23 @@ public class CatalogNameAdaptor { "lakehouse-paimon-3.5", "org.apache.gravitino.spark.connector.paimon.GravitinoPaimonCatalogSpark35"); + private static final Map jdbcCatalogNames = + ImmutableMap.of( + "3.3", + "org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark33", + "3.4", + "org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark34", + "3.5", + "org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark35"); + private static String sparkVersion() { return package$.MODULE$.SPARK_VERSION(); } private static String getCatalogName(String provider, int majorVersion, int minorVersion) { + if (provider.startsWith("jdbc")) { + return jdbcCatalogNames.get(String.format("%d.%d", majorVersion, minorVersion)); + } String key = String.format("%s-%d.%d", provider.toLowerCase(Locale.ROOT), majorVersion, minorVersion); return catalogNames.get(key); diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkCommonIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkCommonIT.java index c7517a3bf82..2eb9e7b9b5a 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkCommonIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkCommonIT.java @@ -119,6 +119,14 @@ private static String getRowLevelDeleteTableSql( protected abstract boolean supportsReplaceColumns(); + protected abstract boolean supportsSchemaAndTableProperties(); + + protected abstract boolean supportsComplexType(); + + protected SparkTableInfoChecker getTableInfoChecker() { + return SparkTableInfoChecker.create(); + } + // Use a custom database not the original default database because SparkCommonIT couldn't // read&write data to tables in default database. The main reason is default database location is // determined by `hive.metastore.warehouse.dir` in hive-site.xml which is local HDFS address @@ -189,6 +197,7 @@ void testLoadCatalogs() { } @Test + @EnabledIf("supportsSchemaAndTableProperties") protected void testCreateAndLoadSchema() { String testDatabaseName = "t_create1"; dropDatabaseIfExists(testDatabaseName); @@ -218,6 +227,7 @@ protected void testCreateAndLoadSchema() { } @Test + @EnabledIf("supportsSchemaAndTableProperties") protected void testAlterSchema() { String testDatabaseName = "t_alter"; dropDatabaseIfExists(testDatabaseName); @@ -266,7 +276,7 @@ void testCreateSimpleTable() { SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(getSimpleTableColumn()) .withComment(null); @@ -287,7 +297,7 @@ void testCreateTableWithDatabase() { createSimpleTable(tableIdentifier); SparkTableInfo tableInfo = getTableInfo(tableIdentifier); SparkTableInfoChecker checker = - SparkTableInfoChecker.create().withName(tableName).withColumns(getSimpleTableColumn()); + getTableInfoChecker().withName(tableName).withColumns(getSimpleTableColumn()); checker.check(tableInfo); checkTableReadWrite(tableInfo); @@ -300,8 +310,7 @@ void testCreateTableWithDatabase() { dropTableIfExists(tableName); createSimpleTable(tableName); tableInfo = getTableInfo(tableName); - checker = - SparkTableInfoChecker.create().withName(tableName).withColumns(getSimpleTableColumn()); + checker = getTableInfoChecker().withName(tableName).withColumns(getSimpleTableColumn()); checker.check(tableInfo); checkTableReadWrite(tableInfo); } @@ -317,7 +326,7 @@ void testCreateTableWithComment() { SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(getSimpleTableColumn()) .withComment(tableComment); @@ -396,6 +405,7 @@ void testListTable() { } @Test + @EnabledIf("supportsSchemaAndTableProperties") void testAlterTableSetAndRemoveProperty() { String tableName = "test_property"; dropTableIfExists(tableName); @@ -425,8 +435,7 @@ void testAlterTableUpdateComment() { "ALTER TABLE %s SET TBLPROPERTIES('%s'='%s')", tableName, ConnectorConstants.COMMENT, comment)); SparkTableInfo tableInfo = getTableInfo(tableName); - SparkTableInfoChecker checker = - SparkTableInfoChecker.create().withName(tableName).withComment(comment); + SparkTableInfoChecker checker = getTableInfoChecker().withName(tableName).withComment(comment); checker.check(tableInfo); } @@ -593,6 +602,7 @@ protected void testAlterTableReplaceColumns() { } @Test + @EnabledIf("supportsComplexType") void testComplexType() { String tableName = "complex_type_table"; dropTableIfExists(tableName); @@ -632,7 +642,7 @@ void testCreateDatasourceFormatPartitionTable() { sql(createTableSQL); SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(getSimpleTableColumn()) .withIdentifyPartition(Arrays.asList("name", "age")); @@ -652,7 +662,7 @@ void testCreateBucketTable() { sql(createTableSQL); SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(getSimpleTableColumn()) .withBucket(4, Arrays.asList("id", "name")); @@ -672,7 +682,7 @@ void testCreateSortBucketTable() { sql(createTableSQL); SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(getSimpleTableColumn()) .withBucket(4, Arrays.asList("id", "name"), Arrays.asList("name", "id")); @@ -695,7 +705,7 @@ void testCreateTableAsSelect() { SparkTableInfo newTableInfo = getTableInfo(newTableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create().withName(newTableName).withColumns(getSimpleTableColumn()); + getTableInfoChecker().withName(newTableName).withColumns(getSimpleTableColumn()); checker.check(newTableInfo); List tableData = getTableData(newTableName); @@ -797,6 +807,7 @@ protected void deleteDirIfExists(String path) { } @Test + @EnabledIf("supportsSchemaAndTableProperties") void testTableOptions() { String tableName = "options_table"; dropTableIfExists(tableName); @@ -806,7 +817,7 @@ void testTableOptions() { SparkTableInfo tableInfo = getTableInfo(tableName); SparkTableInfoChecker checker = - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withTableProperties(ImmutableMap.of(TableCatalog.OPTION_PREFIX + "a", "b")); checker.check(tableInfo); @@ -983,7 +994,7 @@ protected void createSimpleTable(String identifier) { protected void checkTableColumns( String tableName, List columns, SparkTableInfo tableInfo) { - SparkTableInfoChecker.create() + getTableInfoChecker() .withName(tableName) .withColumns(columns) .withComment(null) diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java index b534a9772f7..5bcdc9a2cb6 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java @@ -82,6 +82,7 @@ void startUp() throws Exception { if (lakeHouseIcebergProvider.equalsIgnoreCase(getProvider())) { initIcebergRestServiceEnv(); } + initCatalogEnv(); // Start Gravitino server super.startIntegrationTest(); initHdfsFileSystem(); @@ -151,6 +152,8 @@ private void initHiveEnv() { HiveContainer.HDFS_DEFAULTFS_PORT); } + protected void initCatalogEnv() throws Exception {} + private void initIcebergRestServiceEnv() { ignoreIcebergRestService = false; Map icebergRestServiceConfigs = new HashMap<>(); diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java index b95882a0d01..6ed8e12d647 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java @@ -84,6 +84,16 @@ protected boolean supportsReplaceColumns() { return true; } + @Override + protected boolean supportsSchemaAndTableProperties() { + return true; + } + + @Override + protected boolean supportsComplexType() { + return true; + } + @Test void testCreateHiveFormatPartitionTable() { String tableName = "hive_partition_table"; diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogIT.java index f5fd337a13d..291f8f25dbf 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogIT.java @@ -109,6 +109,16 @@ protected boolean supportsReplaceColumns() { return true; } + @Override + protected boolean supportsSchemaAndTableProperties() { + return true; + } + + @Override + protected boolean supportsComplexType() { + return true; + } + @Override protected String getTableLocation(SparkTableInfo table) { return String.join(File.separator, table.getTableLocation(), "data"); diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT.java new file mode 100644 index 00000000000..1b77047fa8e --- /dev/null +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.spark.connector.integration.test.jdbc; + +import static org.apache.gravitino.integration.test.util.TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.spark.connector.integration.test.SparkCommonIT; +import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfoChecker; +import org.apache.gravitino.spark.connector.jdbc.JdbcPropertiesConstants; +import org.junit.jupiter.api.Tag; + +@Tag("gravitino-docker-test") +public abstract class SparkJdbcMysqlCatalogIT extends SparkCommonIT { + + protected String mysqlUrl; + protected String mysqlUsername; + protected String mysqlPassword; + protected String mysqlDriver; + + @Override + protected boolean supportsSparkSQLClusteredBy() { + return false; + } + + @Override + protected boolean supportsPartition() { + return false; + } + + @Override + protected boolean supportsDelete() { + return false; + } + + @Override + protected boolean supportsSchemaEvolution() { + return false; + } + + @Override + protected boolean supportsReplaceColumns() { + return false; + } + + @Override + protected boolean supportsSchemaAndTableProperties() { + return false; + } + + @Override + protected boolean supportsComplexType() { + return false; + } + + @Override + protected String getCatalogName() { + return "jdbc_mysql"; + } + + @Override + protected String getProvider() { + return "jdbc-mysql"; + } + + @Override + protected SparkTableInfoChecker getTableInfoChecker() { + return SparkJdbcTableInfoChecker.create(); + } + + @Override + protected void initCatalogEnv() throws Exception { + ContainerSuite containerSuite = ContainerSuite.getInstance(); + containerSuite.startMySQLContainer(MYSQL_CATALOG_MYSQL_IT); + mysqlUrl = containerSuite.getMySQLContainer().getJdbcUrl(); + mysqlUsername = containerSuite.getMySQLContainer().getUsername(); + mysqlPassword = containerSuite.getMySQLContainer().getPassword(); + mysqlDriver = containerSuite.getMySQLContainer().getDriverClassName(MYSQL_CATALOG_MYSQL_IT); + } + + @Override + protected Map getCatalogConfigs() { + Map catalogProperties = Maps.newHashMap(); + catalogProperties.put(JdbcPropertiesConstants.GRAVITINO_JDBC_URL, mysqlUrl); + catalogProperties.put(JdbcPropertiesConstants.GRAVITINO_JDBC_USER, mysqlUsername); + catalogProperties.put(JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, mysqlPassword); + catalogProperties.put(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER, mysqlDriver); + return catalogProperties; + } +} diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcTableInfoChecker.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcTableInfoChecker.java new file mode 100644 index 00000000000..32a66923cbe --- /dev/null +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcTableInfoChecker.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.integration.test.jdbc; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfo; +import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfoChecker; + +public class SparkJdbcTableInfoChecker extends SparkTableInfoChecker { + + public static SparkJdbcTableInfoChecker create() { + return new SparkJdbcTableInfoChecker(); + } + + // Spark jdbc table cannot distinguish between comment=null and comment="" + @Override + public SparkTableInfoChecker withColumns(List columns) { + getExpectedTableInfo() + .setColumns( + columns.stream() + .peek( + column -> + column.setComment( + StringUtils.isEmpty(column.getComment()) ? null : column.getComment())) + .collect(Collectors.toList())); + getCheckFields().add(CheckField.COLUMN); + return this; + } + + @Override + public SparkTableInfoChecker withComment(String comment) { + getExpectedTableInfo().setComment(StringUtils.isEmpty(comment) ? "" : comment); + getCheckFields().add(CheckField.COMMENT); + return this; + } +} diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java index 9d036482857..40afa060859 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java @@ -63,6 +63,16 @@ protected boolean supportsSchemaEvolution() { return true; } + @Override + protected boolean supportsSchemaAndTableProperties() { + return true; + } + + @Override + protected boolean supportsComplexType() { + return true; + } + @Override protected boolean supportsReplaceColumns() { // Paimon doesn't support replace columns, because it doesn't support drop all fields in table. diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfo.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfo.java index 077936c29c5..74b3ea09685 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfo.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfo.java @@ -31,6 +31,7 @@ import org.apache.gravitino.spark.connector.ConnectorConstants; import org.apache.gravitino.spark.connector.hive.SparkHiveTable; import org.apache.gravitino.spark.connector.iceberg.SparkIcebergTable; +import org.apache.gravitino.spark.connector.jdbc.SparkJdbcTable; import org.apache.gravitino.spark.connector.paimon.SparkPaimonTable; import org.apache.spark.sql.connector.catalog.SupportsMetadataColumns; import org.apache.spark.sql.connector.catalog.Table; @@ -193,6 +194,8 @@ private static StructType getSchema(Table baseTable) { return ((SparkIcebergTable) baseTable).schema(); } else if (baseTable instanceof SparkPaimonTable) { return ((SparkPaimonTable) baseTable).schema(); + } else if (baseTable instanceof SparkJdbcTable) { + return ((SparkJdbcTable) baseTable).schema(); } else { throw new IllegalArgumentException( "Doesn't support Spark table: " + baseTable.getClass().getName()); diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfoChecker.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfoChecker.java index 33a6a356828..bd7164af786 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfoChecker.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkTableInfoChecker.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import lombok.Data; import org.apache.gravitino.spark.connector.SparkTransformConverter; import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfo.SparkColumnInfo; import org.apache.spark.sql.connector.expressions.Expressions; @@ -34,17 +35,18 @@ * To create an expected SparkTableInfo for verifying the SQL execution result, only the explicitly * set fields will be checked. */ +@Data public class SparkTableInfoChecker { private SparkTableInfo expectedTableInfo = new SparkTableInfo(); private Set checkFields = new LinkedHashSet<>(); - private SparkTableInfoChecker() {} + protected SparkTableInfoChecker() {} public static SparkTableInfoChecker create() { return new SparkTableInfoChecker(); } - private enum CheckField { + protected enum CheckField { NAME, COLUMN, PARTITION, diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkUtilIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkUtilIT.java index ed7d2085ffd..5c188f58001 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkUtilIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/util/SparkUtilIT.java @@ -77,7 +77,7 @@ protected void dropDatabaseIfExists(String database) { // However, Paimon does not support create a database with a specified location. protected void createDatabaseIfNotExists(String database, String provider) { String locationClause = - "lakehouse-paimon".equalsIgnoreCase(provider) + "lakehouse-paimon".equalsIgnoreCase(provider) || provider.startsWith("jdbc") ? "" : String.format("LOCATION '/user/hive/%s'", database); sql(String.format("CREATE DATABASE IF NOT EXISTS %s %s", database, locationClause)); diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java new file mode 100644 index 00000000000..5d3e4d065fd --- /dev/null +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestJdbcPropertiesConverter { + private final JdbcPropertiesConverter jdbcPropertiesConverter = + JdbcPropertiesConverter.getInstance(); + + @Test + void testCatalogProperties() { + String url = "jdbc-url"; + String user = "user1"; + String passwd = "passwd1"; + String driver = "jdbc-driver"; + Map properties = + jdbcPropertiesConverter.toSparkCatalogProperties( + ImmutableMap.of( + JdbcPropertiesConstants.GRAVITINO_JDBC_URL, + url, + JdbcPropertiesConstants.GRAVITINO_JDBC_USER, + user, + JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, + passwd, + JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER, + driver, + "key1", + "value1")); + Assertions.assertEquals( + ImmutableMap.of( + JdbcPropertiesConstants.SPARK_JDBC_URL, + url, + JdbcPropertiesConstants.SPARK_JDBC_USER, + user, + JdbcPropertiesConstants.SPARK_JDBC_PASSWORD, + passwd, + JdbcPropertiesConstants.SPARK_JDBC_DRIVER, + driver), + properties); + } +} diff --git a/spark-connector/v3.3/spark-runtime/build.gradle.kts b/spark-connector/v3.3/spark-runtime/build.gradle.kts index 851473f4bf7..7f4b7f2edcf 100644 --- a/spark-connector/v3.3/spark-runtime/build.gradle.kts +++ b/spark-connector/v3.3/spark-runtime/build.gradle.kts @@ -51,6 +51,7 @@ tasks.withType(ShadowJar::class.java) { relocate("com.google", "org.apache.gravitino.shaded.com.google") relocate("google", "org.apache.gravitino.shaded.google") relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc") + relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") } publishing { diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts index 66c65f863b9..6b633434e46 100644 --- a/spark-connector/v3.3/spark/build.gradle.kts +++ b/spark-connector/v3.3/spark/build.gradle.kts @@ -52,6 +52,9 @@ dependencies { exclude("org.apache.logging.log4j") exclude("org.slf4j") } + testImplementation(project(":catalogs:catalog-jdbc-common")) { + exclude("org.apache.logging.log4j") + } testImplementation(project(":catalogs:hive-metastore-common")) { exclude("*") } @@ -163,6 +166,7 @@ tasks.test { dependsOn(":catalogs:catalog-hive:jar") dependsOn(":iceberg:iceberg-rest-server:jar") dependsOn(":catalogs:catalog-lakehouse-paimon:jar") + dependsOn(":catalogs:catalog-jdbc-mysql:jar") } } diff --git a/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark33.java b/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark33.java new file mode 100644 index 00000000000..d322cd82ca0 --- /dev/null +++ b/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark33.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +public class GravitinoJdbcCatalogSpark33 extends GravitinoJdbcCatalog {} diff --git a/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT33.java b/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT33.java new file mode 100644 index 00000000000..cf190cfd4fb --- /dev/null +++ b/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT33.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.integration.test.jdbc; + +import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark33; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SparkJdbcMysqlCatalogIT33 extends SparkJdbcMysqlCatalogIT { + @Test + void testCatalogClassName() { + String catalogClass = + getSparkSession() + .sessionState() + .conf() + .getConfString("spark.sql.catalog." + getCatalogName()); + Assertions.assertEquals(GravitinoJdbcCatalogSpark33.class.getName(), catalogClass); + } +} diff --git a/spark-connector/v3.4/spark-runtime/build.gradle.kts b/spark-connector/v3.4/spark-runtime/build.gradle.kts index 0b766fef85b..13de294c055 100644 --- a/spark-connector/v3.4/spark-runtime/build.gradle.kts +++ b/spark-connector/v3.4/spark-runtime/build.gradle.kts @@ -51,6 +51,7 @@ tasks.withType(ShadowJar::class.java) { relocate("com.google", "org.apache.gravitino.shaded.com.google") relocate("google", "org.apache.gravitino.shaded.google") relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc") + relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") } publishing { diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts index aa4134a3c71..08ab9ca9caf 100644 --- a/spark-connector/v3.4/spark/build.gradle.kts +++ b/spark-connector/v3.4/spark/build.gradle.kts @@ -53,6 +53,9 @@ dependencies { exclude("org.apache.logging.log4j") exclude("org.slf4j") } + testImplementation(project(":catalogs:catalog-jdbc-common")) { + exclude("org.apache.logging.log4j") + } testImplementation(project(":catalogs:hive-metastore-common")) { exclude("*") } @@ -163,6 +166,7 @@ tasks.test { dependsOn(":catalogs:catalog-hive:jar") dependsOn(":iceberg:iceberg-rest-server:jar") dependsOn(":catalogs:catalog-lakehouse-paimon:jar") + dependsOn(":catalogs:catalog-jdbc-mysql:jar") } } diff --git a/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark34.java b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark34.java new file mode 100644 index 00000000000..e9c091c1882 --- /dev/null +++ b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark34.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import org.apache.gravitino.spark.connector.SparkTableChangeConverter; +import org.apache.gravitino.spark.connector.SparkTableChangeConverter34; +import org.apache.gravitino.spark.connector.SparkTypeConverter; + +public class GravitinoJdbcCatalogSpark34 extends GravitinoJdbcCatalog { + + @Override + protected SparkTypeConverter getSparkTypeConverter() { + return new SparkJdbcTypeConverter34(); + } + + @Override + protected SparkTableChangeConverter getSparkTableChangeConverter( + SparkTypeConverter sparkTypeConverter) { + return new SparkTableChangeConverter34(sparkTypeConverter); + } +} diff --git a/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter34.java b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter34.java new file mode 100644 index 00000000000..bbd32e0225d --- /dev/null +++ b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter34.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; +import org.apache.gravitino.spark.connector.SparkTypeConverter34; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; + +public class SparkJdbcTypeConverter34 extends SparkTypeConverter34 { + @Override + public DataType toSparkType(Type gravitinoType) { + // if spark version lower than 3.4.4, using VarCharType will throw an exception: Unsupported + // type varchar. + if (gravitinoType instanceof Types.VarCharType) { + return DataTypes.StringType; + } else { + return super.toSparkType(gravitinoType); + } + } +} diff --git a/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT34.java b/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT34.java new file mode 100644 index 00000000000..9a4038404d8 --- /dev/null +++ b/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT34.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.spark.connector.integration.test.jdbc; + +import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark34; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SparkJdbcMysqlCatalogIT34 extends SparkJdbcMysqlCatalogIT { + @Test + void testCatalogClassName() { + String catalogClass = + getSparkSession() + .sessionState() + .conf() + .getConfString("spark.sql.catalog." + getCatalogName()); + Assertions.assertEquals(GravitinoJdbcCatalogSpark34.class.getName(), catalogClass); + } +} diff --git a/spark-connector/v3.5/spark-runtime/build.gradle.kts b/spark-connector/v3.5/spark-runtime/build.gradle.kts index 2e21e5bd755..c1cdae6c69d 100644 --- a/spark-connector/v3.5/spark-runtime/build.gradle.kts +++ b/spark-connector/v3.5/spark-runtime/build.gradle.kts @@ -51,6 +51,7 @@ tasks.withType(ShadowJar::class.java) { relocate("com.google", "org.apache.gravitino.shaded.com.google") relocate("google", "org.apache.gravitino.shaded.google") relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc") + relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") } publishing { diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts index 15aa018081d..782d514aed4 100644 --- a/spark-connector/v3.5/spark/build.gradle.kts +++ b/spark-connector/v3.5/spark/build.gradle.kts @@ -53,6 +53,9 @@ dependencies { testImplementation(project(":api")) { exclude("org.apache.logging.log4j") } + testImplementation(project(":catalogs:catalog-jdbc-common")) { + exclude("org.apache.logging.log4j") + } testImplementation(project(":catalogs:hive-metastore-common")) { exclude("*") } @@ -165,6 +168,7 @@ tasks.test { dependsOn(":catalogs:catalog-hive:jar") dependsOn(":iceberg:iceberg-rest-server:jar") dependsOn(":catalogs:catalog-lakehouse-paimon:jar") + dependsOn(":catalogs:catalog-jdbc-mysql:jar") } } diff --git a/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark35.java b/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark35.java new file mode 100644 index 00000000000..1b10d63fa09 --- /dev/null +++ b/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark35.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.jdbc; + +import org.apache.gravitino.spark.connector.SparkTableChangeConverter; +import org.apache.gravitino.spark.connector.SparkTableChangeConverter34; +import org.apache.gravitino.spark.connector.SparkTypeConverter; +import org.apache.gravitino.spark.connector.SparkTypeConverter34; + +public class GravitinoJdbcCatalogSpark35 extends GravitinoJdbcCatalog { + + @Override + protected SparkTypeConverter getSparkTypeConverter() { + return new SparkTypeConverter34(); + } + + @Override + protected SparkTableChangeConverter getSparkTableChangeConverter( + SparkTypeConverter sparkTypeConverter) { + return new SparkTableChangeConverter34(sparkTypeConverter); + } +} diff --git a/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT35.java b/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT35.java new file mode 100644 index 00000000000..00c14e40d2c --- /dev/null +++ b/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT35.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.spark.connector.integration.test.jdbc; + +import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark35; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SparkJdbcMysqlCatalogIT35 extends SparkJdbcMysqlCatalogIT { + @Test + void testCatalogClassName() { + String catalogClass = + getSparkSession() + .sessionState() + .conf() + .getConfString("spark.sql.catalog." + getCatalogName()); + Assertions.assertEquals(GravitinoJdbcCatalogSpark35.class.getName(), catalogClass); + } +}