You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/09/29 01:46:14 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4765 Set
spark.sql.shuffle.partition to 1 for debug on local
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new 67a1c7c KYLIN-4765 Set spark.sql.shuffle.partition to 1 for debug on local
67a1c7c is described below
commit 67a1c7c6c28e1e22f6d4e7c3b20dbbb9b80a52fe
Author: rupengwang <wa...@live.cn>
AuthorDate: Mon Sep 21 12:24:59 2020 +0800
KYLIN-4765 Set spark.sql.shuffle.partition to 1 for debug on local
---
.../org/apache/kylin/engine/spark/application/SparkApplication.java | 3 +++
.../java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index a2e1ad8..3da1271 100644
--- a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -142,6 +142,9 @@ public abstract class SparkApplication {
}
} else if (!isJobOnCluster(sparkConf)) {
sparkConf.set("spark.master", "local");
+ if (!config.getSparkConfigOverride().containsKey("spark.sql.shuffle.partitions")) {
+ sparkConf.set("spark.sql.shuffle.partitions", "1");
+ }
}
// for wrapping credential
diff --git a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
index 4550e19..9e2328c 100644
--- a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
+++ b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
@@ -128,7 +128,6 @@ public class LocalWithSparkSessionTest extends LocalFileMetadataTestCase impleme
sparkConf = new SparkConf().setAppName(UUID.randomUUID().toString()).setMaster("local[4]");
sparkConf.set("spark.serializer", "org.apache.spark.serializer.JavaSerializer");
sparkConf.set(StaticSQLConf.CATALOG_IMPLEMENTATION().key(), "in-memory");
- sparkConf.set("spark.sql.shuffle.partitions", "1");
sparkConf.set("spark.memory.fraction", "0.1");
// opt memory
sparkConf.set("spark.shuffle.detectCorrupt", "false");