[AMORO-3335] Add interface ConfigShade to support encryption of sensitive configuration items and provide a base64 encoding implementation

amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java

@@ -27,6 +27,7 @@
 import org.apache.amoro.api.OptimizingService;
 import org.apache.amoro.config.ConfigHelpers;
 import org.apache.amoro.config.Configurations;
+import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
 import org.apache.amoro.exception.AmoroRuntimeException;
 import org.apache.amoro.server.catalog.CatalogManager;
 import org.apache.amoro.server.catalog.DefaultCatalogManager;
@@ -459,6 +460,8 @@ private void initServiceConfig(Map<String, Object> envConfig) throws Exception {
       // If same configurations in files and environment variables, environment variables have
       // higher priority.
       expandedConfigurationMap.putAll(envConfig);
+      // Decrypt the sensitive configurations if specified
+      expandedConfigurationMap = ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
       serviceConfig = Configurations.fromObjectMap(expandedConfigurationMap);
       AmoroManagementConfValidator.validateConfig(serviceConfig);
       dataSource = DataSourceFactory.createDataSource(serviceConfig);
@@ -533,7 +536,8 @@ private TNonblockingServerSocket getServerSocket(String bindHost, int portNum)
   }
 
   @SuppressWarnings("unchecked")
-  private void expandConfigMap(
+  @VisibleForTesting
+  public static void expandConfigMap(
       Map<String, Object> config, String prefix, Map<String, Object> result) {
     for (Map.Entry<String, Object> entry : config.entrySet()) {
       String key = entry.getKey();

amoro-ams/src/test/java/org/apache/amoro/server/config/TestConfigShade.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config;
+
+import static org.apache.amoro.server.AmoroServiceContainer.expandConfigMap;
+
+import org.apache.amoro.config.shade.impl.Base64ConfigShade;
+import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
+import org.apache.amoro.server.AmoroManagementConf;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
+import org.apache.amoro.shade.guava32.com.google.common.io.Resources;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.amoro.utils.JacksonUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.Map;
+
+public class TestConfigShade {
+  private static final String USERNAME = "admin";
+  private static final String PASSWORD = "password";
+
+  private static final String BASE64_CONFIG_SHADE_IDENTIFIER =
+      new Base64ConfigShade().getIdentifier();
+
+  @Test
+  public void testDecryptOptions() {
+    String encryptUsername = getBase64EncodedText(USERNAME);
+    String encryptPassword = getBase64EncodedText(PASSWORD);
+    Assertions.assertEquals(encryptUsername, "YWRtaW4=");
+    Assertions.assertEquals(encryptPassword, "cGFzc3dvcmQ=");
+
+    String decryptUsername =
+        ConfigShadeUtils.decryptOption(BASE64_CONFIG_SHADE_IDENTIFIER, encryptUsername);
+    String decryptPassword =
+        ConfigShadeUtils.decryptOption(BASE64_CONFIG_SHADE_IDENTIFIER, encryptPassword);
+    Assertions.assertEquals(decryptUsername, USERNAME);
+    Assertions.assertEquals(decryptPassword, PASSWORD);
+  }
+
+  private String getBase64EncodedText(String plaintext) {
+    return Base64.getEncoder().encodeToString(plaintext.getBytes(StandardCharsets.UTF_8));
+  }
+
+  @Test
+  void testDecryptServiceConfigWithDefaultShade() throws Exception {
+    URL resource = Resources.getResource("configs/config-default-shade.yaml");
+    JsonNode yamlConfig =
+        JacksonUtil.fromObjects(
+            new Yaml().loadAs(Files.newInputStream(Paths.get(resource.toURI())), Map.class));
+    Map<String, Object> systemConfig =
+        JacksonUtil.getMap(
+            yamlConfig,
+            AmoroManagementConf.SYSTEM_CONFIG,
+            new TypeReference<Map<String, Object>>() {});
+    Map<String, Object> expandedConfigurationMap = Maps.newHashMap();
+    expandConfigMap(systemConfig, "", expandedConfigurationMap);
+    expandedConfigurationMap = ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
+
+    Assertions.assertEquals(decryptedServiceConfigWithDefaultShade, expandedConfigurationMap);
+  }
+
+  @Test
+  void testDecryptServiceConfigWithBase64Shade() throws Exception {
+    URL resource = Resources.getResource("configs/config-base64-shade.yaml");
+    JsonNode yamlConfig =
+        JacksonUtil.fromObjects(
+            new Yaml().loadAs(Files.newInputStream(Paths.get(resource.toURI())), Map.class));
+    Map<String, Object> systemConfig =
+        JacksonUtil.getMap(
+            yamlConfig,
+            AmoroManagementConf.SYSTEM_CONFIG,
+            new TypeReference<Map<String, Object>>() {});
+    Map<String, Object> expandedConfigurationMap = Maps.newHashMap();
+    expandConfigMap(systemConfig, "", expandedConfigurationMap);
+    expandedConfigurationMap = ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
+
+    Assertions.assertEquals(decryptedServiceConfigWithBase64Shade, expandedConfigurationMap);
+  }
+
+  private final Map<String, String> decryptedServiceConfigWithDefaultShade =
+      ImmutableMap.<String, String>builder()
+          .put("admin-username", "admin")
+          .put("admin-password", "admin")
+          .put("server-bind-host", "0.0.0.0")
+          .put("server-expose-host", "127.0.0.1")
+          .put("shade.identifier", "default")
+          .put("database.type", "mysql")
+          .put("database.jdbc-driver-class", "com.mysql.cj.jdbc.Driver")
+          .put(
+              "database.url",
+              "jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false")
+          .put("database.username", "root")
+          .put("database.password", "password")
+          .build();
+
+  private final Map<String, String> decryptedServiceConfigWithBase64Shade =
+      ImmutableMap.<String, String>builder()
+          .put("admin-username", "admin")
+          .put("admin-password", "admin")
+          .put("server-bind-host", "0.0.0.0")
+          .put("server-expose-host", "127.0.0.1")
+          .put("shade.identifier", "base64")
+          .put("shade.sensitive-keywords", "admin-password;database.password")
+          .put("database.type", "mysql")
+          .put("database.jdbc-driver-class", "com.mysql.cj.jdbc.Driver")
+          .put(
+              "database.url",
+              "jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false")
+          .put("database.username", "root")
+          .put("database.password", "password")
+          .build();
+}

amoro-ams/src/test/resources/configs/config-base64-shade.yaml

@@ -0,0 +1,38 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+################################################################################
+
+ams:
+  admin-username: admin
+  admin-password: YWRtaW4=
+  server-bind-host: "0.0.0.0"
+  server-expose-host: "127.0.0.1"
+
+  shade:
+    identifier: base64
+    sensitive-keywords: admin-password;database.password
+
+  database:
+    type: mysql
+    jdbc-driver-class: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
+    username: root
+    password: cGFzc3dvcmQ=
+
+containers:
+  - name: localContainer
+    container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer

amoro-ams/src/test/resources/configs/config-default-shade.yaml

@@ -0,0 +1,37 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+################################################################################
+
+ams:
+  admin-username: admin
+  admin-password: admin
+  server-bind-host: "0.0.0.0"
+  server-expose-host: "127.0.0.1"
+
+  shade:
+    identifier: default
+
+  database:
+    type: mysql
+    jdbc-driver-class: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
+    username: root
+    password: password
+
+containers:
+  - name: localContainer
+    container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer

amoro-common/src/main/java/org/apache/amoro/config/shade/ConfigShade.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config.shade;
+
+import org.apache.amoro.config.Configurations;
+
+/**
+ * The interface that provides the ability to decrypt {@link
+ * org.apache.amoro.config.Configurations}.
+ */
+public interface ConfigShade {
+  /**
+   * Initializes the custom instance using the AMS configuration.
+   *
+   * <p>This method can be useful when decryption requires an external file (e.g. a key file)
+   * defined in the service configs.
+   */
+  default void initialize(Configurations serviceConfig) throws Exception {}
+
+  /**
+   * The unique identifier of the current interface, used it to select the expected {@link
+   * ConfigShade}.
+   */
+  String getIdentifier();
+
+  /**
+   * Decrypt the content.
+   *
+   * @param content The content to decrypt
+   */
+  String decrypt(String content);
+}

amoro-common/src/main/java/org/apache/amoro/config/shade/impl/Base64ConfigShade.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config.shade.impl;
+
+import org.apache.amoro.config.shade.ConfigShade;
+
+import java.util.Base64;
+
+/** Base64 ConfigShade. */
+public class Base64ConfigShade implements ConfigShade {
+
+  private static final Base64.Decoder DECODER = Base64.getDecoder();
+
+  private static final String IDENTIFIER = "base64";
+
+  @Override
+  public String getIdentifier() {
+    return IDENTIFIER;
+  }
+
+  @Override
+  public String decrypt(String content) {
+    return new String(DECODER.decode(content));
+  }
+}