[GitHub] [linkis] peacewong commented on a diff in pull request #4850: spark support yarn cluster

["peacewong (via GitHub)" <gi...@apache.org>]

peacewong commented on code in PR #4850:
URL: https://github.com/apache/linkis/pull/4850#discussion_r1290868770


##########
linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala:
##########
@@ -61,7 +61,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging {
     newMap.put("spring.mvc.servlet.path", ServerConfiguration.BDP_SERVER_RESTFUL_URI.getValue)
     DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(newMap.toMap))
 
-    val engineConnPidCallBack = new EngineConnPidCallback()
+    val engineConnPidCallBack = new EngineConnIdentifierCallback()

Review Comment:
   val engineConnIdentifierCallback= new EngineConnIdentifierCallback()



##########
linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala:
##########
@@ -144,19 +144,33 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging
     val master =
       sparkConf.getOption("spark.master").getOrElse(CommonVars("spark.master", "yarn").getValue)
     logger.info(s"------ Create new SparkContext {$master} -------")
-    val pysparkBasePath = SparkConfiguration.SPARK_HOME.getValue
-    val pysparkPath = new File(pysparkBasePath, "python" + File.separator + "lib")
-    var pythonLibUris = pysparkPath.listFiles().map(_.toURI.toString).filter(_.endsWith(".zip"))
-    if (pythonLibUris.length == 2) {
-      val sparkConfValue1 = Utils.tryQuietly(CommonVars("spark.yarn.dist.files", "").getValue)
-      val sparkConfValue2 = Utils.tryQuietly(sparkConf.get("spark.yarn.dist.files"))
-      if (StringUtils.isNotBlank(sparkConfValue2)) {
-        pythonLibUris = sparkConfValue2 +: pythonLibUris
-      }
-      if (StringUtils.isNotBlank(sparkConfValue1)) {
-        pythonLibUris = sparkConfValue1 +: pythonLibUris
+
+    // Set deploy-mode with the optional values `cluster`and `client`, the default value `client`
+    val deployMode: String = SPARK_DEPLOY_MODE.getValue(options)

Review Comment:
   should to use label judge



##########
linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala:
##########
@@ -146,11 +148,19 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w
       throw t
     }
     LoggerUtils.removeJobIdMDC()
+    val deployMode: String =
+      request.creationDesc.properties.getOrDefault("spark.submit.deployMode", "client")

Review Comment:
   should judge  by yarn cluster mode label 



##########
linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala:
##########
@@ -70,9 +71,17 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long)
 
   private var thread: Thread = _
 
+  private var applicationId: String = _

Review Comment:
   should applicationId = sc.applicationId



##########
linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala:
##########
@@ -61,7 +66,7 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa
     val executorMemory = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_MEMORY)
     val numExecutors = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_INSTANCES)
 
-    val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath)
+    var files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath)

Review Comment:
   update to array buffer



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: notifications-unsubscribe@linkis.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@linkis.apache.org
For additional commands, e-mail: notifications-help@linkis.apache.org