Merge branch 'main' into issue-5900

.github/workflows/access-control-integration-test.yml

@@ -92,7 +92,7 @@ jobs:
           ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: authorizations-integrate-test-reports-${{ matrix.java-version }}

.github/workflows/backend-integration-test-action.yml

@@ -64,7 +64,7 @@ jobs:
           -x :authorizations:authorization-chain:test -x :authorizations:authorization-ranger:test 
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}

.github/workflows/build.yml

@@ -90,7 +90,7 @@ jobs:
           ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false
 
       - name: Upload unit tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: unit test report
@@ -129,7 +129,7 @@ jobs:
         run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build
 
       - name: Upload unit tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: unit test report

.github/workflows/cron-integration-test.yml

@@ -87,7 +87,7 @@ jobs:
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
         with:
           name: integrate test reports

.github/workflows/docker-image.yml

@@ -32,6 +32,11 @@ on:
         description: 'Publish Docker token'
         required: true
         type: string
+      publish-latest-tag:
+        description: 'Whether to update the latest tag. This operation is only applicable to official releases and should not be used for Release Candidate (RC).'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   publish-docker-image:
@@ -83,6 +88,12 @@ jobs:
             echo "image_type=iceberg-rest-server" >> $GITHUB_ENV
             echo "image_name=apache/gravitino-iceberg-rest" >> $GITHUB_ENV
           fi
+          
+          if [ "${{ github.event.inputs.publish-latest-tag }}" == "true" ]; then
+            echo "publish_latest=true" >> $GITHUB_ENV
+          else
+            echo "publish_latest=false" >> $GITHUB_ENV
+          fi 
 
       - name: Check publish Docker token
         run: |
@@ -115,8 +126,16 @@ jobs:
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/hostedtoolcache/CodeQL
           
-          if [[ "${image_type}" == "gravitino" || "${image_type}" == "iceberg-rest-server" ]]; then
-            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.version }} --latest
+          if [[ -n "${tag_name}" ]]; then
+            full_tag_name="${tag_name}-${{ github.event.inputs.version }}"
+          else
+            full_tag_name="${{ github.event.inputs.version }}"
+          fi
+          
+          if [[ "${publish_latest}" == "true" ]]; then
+            echo "Publish tag ${full_tag_name}, and update latest too."
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} --latest
           else
-            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag "${tag_name}-${{ github.event.inputs.version }}"
-          fi
+            echo "Publish tag ${full_tag_name}."
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name}
+          fi

.github/workflows/flink-integration-test-action.yml

@@ -52,7 +52,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: flink-connector-integrate-test-reports-${{ inputs.java-version }}

.github/workflows/frontend-integration-test.yml

@@ -88,7 +88,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: integrate-test-reports-${{ matrix.java-version }}

.github/workflows/gvfs-fuse-build-test.yml

@@ -88,7 +88,7 @@ jobs:
           dev/ci/util_free_space.sh
 
       - name: Upload tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }}

.github/workflows/python-integration-test.yml

@@ -82,7 +82,7 @@ jobs:
           done
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
         with:
           name: integrate test reports

.github/workflows/spark-integration-test-action.yml

@@ -63,7 +63,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: spark-connector-integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}

.github/workflows/trino-integration-test.yml

@@ -90,7 +90,7 @@ jobs:
           trino-connector/integration-test/trino-test-tools/run_test.sh
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: trino-connector-integrate-test-reports-${{ matrix.java-version }}

.gitignore

@@ -27,6 +27,7 @@ replay_pid*
 **/.gradle
 **/.idea
 !/.idea/icon.svg
+!.idea/vcs.xml
 **/build
 gen
 **/.DS_Store

api/src/main/java/org/apache/gravitino/MetadataObjects.java

@@ -126,6 +126,7 @@ public static MetadataObject parent(MetadataObject object) {
       case TABLE:
       case FILESET:
       case TOPIC:
+      case MODEL:
         parentType = MetadataObject.Type.SCHEMA;
         break;
       case SCHEMA:

authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java

@@ -122,8 +122,13 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "MetadataObject: [fullName=" + fullName() + "],  [path=" + path == null
-        ? "null"
-        : path + "], [type=" + type + "]";
+    String strPath = path == null ? "null" : path;
+    return "MetadataObject: [fullName="
+        + fullName()
+        + "],  [path="
+        + strPath
+        + "], [type="
+        + type
+        + "]";
   }
 }

authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java

@@ -110,8 +110,13 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime
   @Override
   public Boolean onRoleCreated(Role role) throws AuthorizationPluginException {
     List<String> sqls = getCreateRoleSQL(role.name());
+    boolean createdNewly = false;
     for (String sql : sqls) {
-      executeUpdateSQL(sql, "already exists");
+      createdNewly = executeUpdateSQL(sql, "already exists");
+    }
+
+    if (!createdNewly) {
+      return true;
     }
 
     if (role.securableObjects() != null) {
@@ -140,7 +145,6 @@ public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException {
   @Override
   public Boolean onRoleUpdated(Role role, RoleChange... changes)
       throws AuthorizationPluginException {
-    onRoleCreated(role);
     for (RoleChange change : changes) {
       if (change instanceof RoleChange.AddSecurableObject) {
         SecurableObject object = ((RoleChange.AddSecurableObject) change).getSecurableObject();
@@ -381,14 +385,15 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti
         "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode());
   }
 
-  public void executeUpdateSQL(String sql, String ignoreErrorMsg) {
+  public boolean executeUpdateSQL(String sql, String ignoreErrorMsg) {
     try (final Connection connection = getConnection()) {
       try (final Statement statement = connection.createStatement()) {
         statement.executeUpdate(sql);
+        return true;
       }
     } catch (SQLException se) {
       if (ignoreErrorMsg != null && se.getMessage().contains(ignoreErrorMsg)) {
-        return;
+        return false;
       }
       LOG.error("JDBC authorization plugin exception: ", se);
       throw toAuthorizationPluginException(se);

authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java

@@ -47,4 +47,31 @@ public void PathBasedMetadataObjectNotEquals() {
 
     Assertions.assertNotEquals(pathBasedMetadataObject1, pathBasedMetadataObject2);
   }
+
+  @Test
+  void testToString() {
+    PathBasedMetadataObject pathBasedMetadataObject1 =
+        new PathBasedMetadataObject("parent", "name", "path", PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=parent.name],  [path=path], [type=PATH]",
+        pathBasedMetadataObject1.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject2 =
+        new PathBasedMetadataObject("parent", "name", null, PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=parent.name],  [path=null], [type=PATH]",
+        pathBasedMetadataObject2.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject3 =
+        new PathBasedMetadataObject(null, "name", null, PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=name],  [path=null], [type=PATH]",
+        pathBasedMetadataObject3.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject4 =
+        new PathBasedMetadataObject(null, "name", "path", PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=name],  [path=path], [type=PATH]",
+        pathBasedMetadataObject4.toString());
+  }
 }

authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java

@@ -74,9 +74,10 @@ public List<String> getSetOwnerSQL(
           return Collections.emptyList();
         }
 
-        public void executeUpdateSQL(String sql, String ignoreErrorMsg) {
+        public boolean executeUpdateSQL(String sql, String ignoreErrorMsg) {
           Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql);
           currentSQLIndex++;
+          return true;
         }
       };
 
@@ -148,23 +149,21 @@ public void testPermissionManagement() {
 
     // Test metalake object and different role change
     resetSQLIndex();
-    expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp");
+    expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE *.* TO ROLE tmp");
     SecurableObject metalakeObject =
         SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.SelectTable.allow()));
     RoleChange roleChange = RoleChange.addSecurableObject("tmp", metalakeObject);
     plugin.onRoleUpdated(role, roleChange);
 
     resetSQLIndex();
-    expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "REVOKE SELECT ON TABLE *.* FROM ROLE tmp");
+    expectSQLs = Lists.newArrayList("REVOKE SELECT ON TABLE *.* FROM ROLE tmp");
     roleChange = RoleChange.removeSecurableObject("tmp", metalakeObject);
     plugin.onRoleUpdated(role, roleChange);
 
     resetSQLIndex();
     expectSQLs =
         Lists.newArrayList(
-            "CREATE ROLE tmp",
-            "REVOKE SELECT ON TABLE *.* FROM ROLE tmp",
-            "GRANT CREATE ON TABLE *.* TO ROLE tmp");
+            "REVOKE SELECT ON TABLE *.* FROM ROLE tmp", "GRANT CREATE ON TABLE *.* TO ROLE tmp");
     SecurableObject newMetalakeObject =
         SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.CreateTable.allow()));
     roleChange = RoleChange.updateSecurableObject("tmp", metalakeObject, newMetalakeObject);
@@ -175,7 +174,7 @@ public void testPermissionManagement() {
     SecurableObject catalogObject =
         SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.SelectTable.allow()));
     roleChange = RoleChange.addSecurableObject("tmp", catalogObject);
-    expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp");
+    expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE *.* TO ROLE tmp");
     plugin.onRoleUpdated(role, roleChange);
 
     // Test schema object
@@ -184,8 +183,7 @@ public void testPermissionManagement() {
         SecurableObjects.ofSchema(
             catalogObject, "schema", Lists.newArrayList(Privileges.SelectTable.allow()));
     roleChange = RoleChange.addSecurableObject("tmp", schemaObject);
-    expectSQLs =
-        Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.* TO ROLE tmp");
+    expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE schema.* TO ROLE tmp");
     plugin.onRoleUpdated(role, roleChange);
 
     // Test table object
@@ -194,8 +192,18 @@ public void testPermissionManagement() {
         SecurableObjects.ofTable(
             schemaObject, "table", Lists.newArrayList(Privileges.SelectTable.allow()));
     roleChange = RoleChange.addSecurableObject("tmp", tableObject);
-    expectSQLs =
-        Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.table TO ROLE tmp");
+    expectSQLs = Lists.newArrayList("GRANT SELECT ON TABLE schema.table TO ROLE tmp");
+    plugin.onRoleUpdated(role, roleChange);
+
+    // Test the role with objects
+    resetSQLIndex();
+    role =
+        RoleEntity.builder()
+            .withId(-1L)
+            .withName("tmp")
+            .withSecurableObjects(Lists.newArrayList(tableObject))
+            .withAuditInfo(AuditInfo.EMPTY)
+            .build();
     plugin.onRoleUpdated(role, roleChange);
   }
 

catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java

@@ -33,6 +33,7 @@ public class IcebergPropertiesUtils {
   // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will
   // change it to `catalogType` automatically and pass it to Iceberg.
   public static final Map<String, String> GRAVITINO_CONFIG_TO_ICEBERG;
+  public static final Map<String, String> ICEBERG_CATALOG_CONFIG_TO_GRAVITINO;
 
   static {
     Map<String, String> map = new HashMap();
@@ -65,6 +66,14 @@ public class IcebergPropertiesUtils {
         AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
         IcebergConstants.ICEBERG_ADLS_STORAGE_ACCOUNT_KEY);
     GRAVITINO_CONFIG_TO_ICEBERG = Collections.unmodifiableMap(map);
+
+    Map<String, String> icebergCatalogConfigToGravitino = new HashMap<>();
+    map.forEach(
+        (key, value) -> {
+          icebergCatalogConfigToGravitino.put(value, key);
+        });
+    ICEBERG_CATALOG_CONFIG_TO_GRAVITINO =
+        Collections.unmodifiableMap(icebergCatalogConfigToGravitino);
   }
 
   /**

catalogs/catalog-hive/build.gradle.kts

@@ -158,6 +158,8 @@ tasks {
       exclude("guava-*.jar")
       exclude("log4j-*.jar")
       exclude("slf4j-*.jar")
+      // Exclude the following jars to avoid conflict with the jars in authorization-gcp
+      exclude("protobuf-java-*.jar")
     }
     into("$rootDir/distribution/package/catalogs/hive/libs")
   }

clients/cli/src/main/java/org/apache/gravitino/cli/AreYouSure.java

@@ -32,16 +32,20 @@ public class AreYouSure {
    * @return {@code true} if the action is to continue {@code false} otherwise.
    */
   public static boolean really(boolean force) {
-    Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8.name());
-
     /* force option for scripting */
     if (force) {
       return true;
     }
 
-    System.out.println(
-        "This command could result in data loss or other issues. Are you sure you want to do this? (Y/N)");
-    String answer = scanner.next();
-    return answer.equals("Y");
+    try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8.name())) {
+      System.out.println(
+          "This command could result in data loss or other issues. Are you sure you want to do this? (Y/N)");
+      String answer = scanner.next();
+      return "Y".equals(answer);
+    } catch (Exception e) {
+      System.err.println("Error while reading user input: " + e.getMessage());
+      Main.exit(-1);
+    }
+    return false;
   }
 }