You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2019/05/06 02:40:44 UTC
[incubator-amaterasu] 28/36: virtualenv isn't supported by spark,
provided alternatives
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 8043d0eaafe6cf13ffe2bac0b2733a4e74fd67e9
Author: Nadav Har Tzvi <na...@apache.org>
AuthorDate: Sat Apr 27 12:30:58 2019 +0300
virtualenv isn't supported by spark, provided alternatives
---
.../spark/dispatcher/runners/providers/PySparkRunnerProvider.scala | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
index 7cac602..61fa555 100644
--- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
+++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -14,10 +14,8 @@ class PySparkRunnerProvider(val env: String, val conf: ClusterConfig) extends Py
// s" && env PYSPARK_PYTHON=$getVirtualPythonPath" +
//s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + d
s" && $$SPARK_HOME/bin/spark-submit --master yarn-client " +
- s"--conf spark.pyspark.virtualenv.enabled=true " +
- s"--conf spark.pyspark.virtualenv.type=native " +
- s"--conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin " +
- s"--conf spark.pyspark.python=$getVirtualPythonPath " +
+ s"--conf spark.pyspark.python=${conf.pythonPath}" +
+ s"--conf spark.pyspark.driver.python=$getVirtualPythonPath " +
s"--files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}"
}