Merge pull request #511 from InterestingLab/rickyhuo.enhance.driver

Rickyhuo.enhance.driverConf
apache · Jun 8, 2020 · 54c6953 · 54c6953
2 parents 46bc907 + 99bb2cc
commit 54c6953
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 4 deletions.
diff --git a/bin/start-waterdrop.sh b/bin/start-waterdrop.sh
@@ -109,7 +109,7 @@ string_trim() {
 
 variables_substitution=$(string_trim "${variables_substitution}")
 
-## get spark conf from config file and specify them in spark-submit
+## get spark conf from config file and specify them in spark-submit --conf
 function get_spark_conf {
     spark_conf=$(java ${variables_substitution} -cp ${assemblyJarName} io.github.interestinglab.waterdrop.config.ExposeSparkConf ${CONFIG_FILE})
     if [ "$?" != "0" ]; then
@@ -119,9 +119,21 @@ function get_spark_conf {
     echo ${spark_conf}
 }
 
-sparkconf=$(get_spark_conf)
+sparkConf=$(get_spark_conf)
 
-echo "[INFO] spark conf: ${sparkconf}"
+echo "[INFO] spark conf: ${sparkConf}"
+
+## get spark driver conf from config file and specify them in spark-submit
+function get_spark_driver_conf {
+    spark_conf=$(java ${variables_substitution} -cp ${assemblyJarName} io.github.interestinglab.waterdrop.config.ExposeSparkDriverConf ${CONFIG_FILE})
+    if [ "$?" != "0" ]; then
+        echo "[ERROR] config file does not exists or cannot be parsed due to invalid format"
+        exit -1
+    fi
+    echo ${spark_conf}
+}
+
+sparkDriverConf=$(get_spark_driver_conf)
 
 # Spark Driver Options
 driverJavaOpts=""
@@ -159,11 +171,12 @@ exec ${SPARK_HOME}/bin/spark-submit --class io.github.interestinglab.waterdrop.W
     --name $(getAppName ${CONFIG_FILE}) \
     --master ${MASTER} \
     --deploy-mode ${DEPLOY_MODE} \
+    ${sparkDriverConf} \
     --queue "${QUEUE}" \
     --driver-java-options "${clientModeDriverJavaOpts}" \
     --conf spark.executor.extraJavaOptions="${executorJavaOpts}" \
     --conf spark.driver.extraJavaOptions="${driverJavaOpts}" \
-    ${sparkconf} \
+    ${sparkConf} \
     ${JarDepOpts} \
     ${FilesDepOpts} \
     ${assemblyJarName} ${CMD_ARGUMENTS}
diff --git a/...p-core/src/main/java/io/github/interestinglab/waterdrop/config/ExposeSparkDriverConf.java b/...p-core/src/main/java/io/github/interestinglab/waterdrop/config/ExposeSparkDriverConf.java
@@ -0,0 +1,53 @@
+package io.github.interestinglab.waterdrop.config;
+
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.io.File;
+
+
+public class ExposeSparkDriverConf {
+
+    public static List<String> splitKey(String key) {
+        List<String> keys = new LinkedList<>();
+        int index = 0;
+        for (int i = 0; i < key.length(); i ++) {
+            char symbol = key.charAt(i);
+            if (symbol >= 'A' && symbol <= 'Z') {
+                keys.add(key.substring(index, i).toLowerCase());
+                index = i;
+            }
+        }
+
+        keys.add(key.substring(index).toLowerCase());
+
+        if (keys.get(0).equals("extra")) {
+            return keys.subList(1, keys.size());
+        }
+        return keys;
+    }
+
+    public static void main(String[] args) throws Exception {
+        Config appConfig = ConfigFactory.parseFile(new File(args[0]))
+                .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+                .resolveWith(ConfigFactory.systemProperties(), ConfigResolveOptions.defaults().setAllowUnresolved(true));
+
+        String driverPrefix = "spark.driver.";
+        Config sparkConfig = appConfig.getConfig("spark");
+
+        if (!TypesafeConfigUtils.hasSubConfig(sparkConfig, driverPrefix)) {
+            System.out.println("");
+        } else {
+            Config sparkDriverConfig = TypesafeConfigUtils.extractSubConfig(sparkConfig, driverPrefix, false);
+            StringBuilder stringBuilder = new StringBuilder();
+            for (Map.Entry<String, ConfigValue> entry: sparkDriverConfig.entrySet()) {
+                List<String> keys = splitKey(entry.getKey());
+                String conf = String.format(" --driver-%s=%s ", String.join("-", keys), entry.getValue().unwrapped());
+                stringBuilder.append(conf);
+            }
+
+            System.out.println(stringBuilder.toString());
+        }
+    }
+}