You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2019/05/06 02:40:37 UTC
[incubator-amaterasu] 21/36: fixing virtual env config
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 667bb98319cce4bce4b8a4a769f72fb246f12d91
Author: Yaniv Rodenski <ya...@shinto.io>
AuthorDate: Fri Apr 26 14:59:00 2019 +1000
fixing virtual env config
---
.../python/dispatcher/runners/providers/PythonRunnerProviderBase.kt | 1 +
.../spark/dispatcher/runners/providers/PySparkRunnerProvider.scala | 6 +++---
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt
index 0f91d3a..59f3896 100644
--- a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt
+++ b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt
@@ -27,6 +27,7 @@ abstract class PythonRunnerProviderBase(val env: String, val conf: ClusterConfig
private val requirementsFileName: String = "ama-requirements.txt"
private val mandatoryPYPIPackages: Array<String> = arrayOf("requests")
protected val virtualPythonPath = "amaterasu_env/bin/python"
+ protected val virtualPythonBin = "amaterasu_env/bin"
override val runnerResources: Array<String>
get() = arrayOf("amaterasu-sdk-${conf.version()}.zip")
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
index 60e0700..f4ddb6e 100644
--- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
+++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -11,9 +11,9 @@ class PySparkRunnerProvider(val env: String, val conf: ClusterConfig) extends Py
log.info(s"===> Cluster manager: ${conf.mode}")
command +
//s" $$SPARK_HOME/conf/spark-env.sh" +
- s" && env PYSPARK_PYTHON=$getVirtualPythonPath" +
- s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" +
- s" $$SPARK_HOME/bin/spark-submit --master yarn-cluster --files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}"
+ // s" && env PYSPARK_PYTHON=$getVirtualPythonPath" +
+ //s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + d
+ s" $$SPARK_HOME/bin/spark-submit --master yarn-cluster --conf spark.pyspark.virtualenv.enabled=true --conf spark.pyspark.virtualenv.type=native--conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin --conf spark.pyspark.python=$getVirtualPythonPath --files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}"
}
override def getRunnerResources: Array[String] = {