You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/02/07 14:09:55 UTC
[kylin] 01/02: KYLIN-4889, fix spark engine in local mode
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 591c0e61e5cdd74b2bce2d36d5b3995130673559
Author: feng.zhu <fi...@outlook.com>
AuthorDate: Fri Jan 29 12:07:01 2021 +0800
KYLIN-4889, fix spark engine in local mode
---
.../scala/org/apache/spark/sql/KylinSession.scala | 3 +++
.../org/apache/spark/sql/SparderContext.scala | 31 +++++++++-------------
2 files changed, 16 insertions(+), 18 deletions(-)
diff --git a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
index e3c532e..892cf9c 100644
--- a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
+++ b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
@@ -199,6 +199,9 @@ object KylinSession extends Logging {
}
sparkConf.set("spark.yarn.am.extraJavaOptions",
s"$yarnAMJavaOptions $amKerberosConf")
+ } else {
+ // in case spark conf is overridden by kylinconfig
+ sparkConf.setMaster("local")
}
sparkConf
diff --git a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
index 0d25dba..d4d6f49 100644
--- a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
+++ b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
@@ -131,26 +131,21 @@ object SparderContext extends Logging {
override def run(): Unit = {
try {
val kylinConf: KylinConfig = KylinConfig.getInstanceFromEnv
- val sparkSession = System.getProperty("spark.local") match {
+ val master = System.getProperty("spark.local") match {
case "true" =>
- SparkSession.builder
- .master("local")
- .appName(kylinConf.getSparderAppName)
- .withExtensions { ext =>
- ext.injectPlannerStrategy(_ => KylinSourceStrategy)
- }
- .enableHiveSupport()
- .getOrCreateKylinSession()
+ "local"
case _ =>
- SparkSession.builder
- .appName(kylinConf.getSparderAppName)
- .master("yarn-client")
- .withExtensions { ext =>
- ext.injectPlannerStrategy(_ => KylinSourceStrategy)
- }
- .enableHiveSupport()
- .getOrCreateKylinSession()
+ "yarn-client"
}
+ val sparkSession = SparkSession.builder
+ .master(master)
+ .appName(kylinConf.getSparderAppName)
+ .withExtensions { ext =>
+ ext.injectPlannerStrategy(_ => KylinSourceStrategy)
+ }
+ .enableHiveSupport()
+ .getOrCreateKylinSession()
+
if (kylinConf.isKylinMetricsReporterForQueryEnabled) {
val appStatusListener = new SparderMetricsListener()
sparkSession.sparkContext.addSparkListener(appStatusListener)
@@ -177,7 +172,7 @@ object SparderContext extends Logging {
.getContextClassLoader
.toString)
initMonitorEnv()
- System.getProperty("spark.local") match {
+ master match {
case "true" =>
master_app_url = "http://localhost:" + sparkSession.sparkContext.getConf
.get("spark.ui.port", "4040")