[GitHub] [zeppelin] Reamer commented on a diff in pull request #4398: [ZEPPELIN-5760] fix passing configs to spark in k8s

[GitBox <gi...@apache.org>]

Reamer commented on code in PR #4398:
URL: https://github.com/apache/zeppelin/pull/4398#discussion_r960708753


##########
zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java:
##########
@@ -403,28 +420,35 @@ boolean isSparkOnKubernetes(Properties interpreterProperties) {
   }
 
   @VisibleForTesting
-  String buildSparkSubmitOptions(String userName) {
-    StringBuilder options = new StringBuilder();
-
-    options.append(" --master k8s://https://kubernetes.default.svc");
-    options.append(" --deploy-mode client");
-    if (properties.containsKey(SPARK_DRIVER_MEMORY)) {
-      options.append(" --driver-memory ").append(properties.get(SPARK_DRIVER_MEMORY));
-    }
+  String prepareZeppelinSparkConf(String userName) {
+    StringJoiner sparkConfSJ = new StringJoiner("|");
     if (isUserImpersonated() && !StringUtils.containsIgnoreCase(userName, "anonymous")) {
-      options.append(" --proxy-user ").append(userName);
+      sparkConfSJ.add("--proxy-user");
+      sparkConfSJ.add(userName);
     }
-    options.append(" --conf spark.kubernetes.namespace=").append(getInterpreterNamespace());
-    options.append(" --conf spark.executor.instances=1");
-    options.append(" --conf spark.kubernetes.driver.pod.name=").append(getPodName());
-    String sparkContainerImage = properties.containsKey(SPARK_CONTAINER_IMAGE) ? properties.getProperty(SPARK_CONTAINER_IMAGE) : sparkImage;
-    options.append(" --conf spark.kubernetes.container.image=").append(sparkContainerImage);
-    options.append(" --conf spark.driver.bindAddress=0.0.0.0");
-    options.append(" --conf spark.driver.host=").append(getInterpreterPodDnsName());
-    options.append(" --conf spark.driver.port=").append(getSparkDriverPort());
-    options.append(" --conf spark.blockManager.port=").append(getSparkBlockManagerPort());
-
-    return options.toString();
+
+    for (String key : properties.stringPropertyNames()) {
+      String propValue = properties.getProperty(key);
+      if (isSparkConf(key, propValue)) {
+        sparkConfSJ.add("--conf");

Review Comment:
   Here the same error.
   desired is "--conf spark=123|--conf spark.1=123"



##########
zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java:
##########
@@ -403,28 +420,35 @@ boolean isSparkOnKubernetes(Properties interpreterProperties) {
   }
 
   @VisibleForTesting
-  String buildSparkSubmitOptions(String userName) {
-    StringBuilder options = new StringBuilder();
-
-    options.append(" --master k8s://https://kubernetes.default.svc");
-    options.append(" --deploy-mode client");
-    if (properties.containsKey(SPARK_DRIVER_MEMORY)) {
-      options.append(" --driver-memory ").append(properties.get(SPARK_DRIVER_MEMORY));
-    }
+  String prepareZeppelinSparkConf(String userName) {
+    StringJoiner sparkConfSJ = new StringJoiner("|");
     if (isUserImpersonated() && !StringUtils.containsIgnoreCase(userName, "anonymous")) {
-      options.append(" --proxy-user ").append(userName);
+      sparkConfSJ.add("--proxy-user");
+      sparkConfSJ.add(userName);

Review Comment:
   Possible error.
   This creates "--proxy-user|user". desired is "--proxy-user|user"



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscribe@zeppelin.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org