You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/08/27 07:37:22 UTC
[kylin] 05/08: KYLIN-4660 remove deprecated config
'spark.yarn.executor.memoryOverhead'
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 621868cb04d03df9683bb4c2508afc92f23058ad
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Wed Aug 26 00:23:19 2020 +0800
KYLIN-4660 remove deprecated config 'spark.yarn.executor.memoryOverhead'
---
core-common/src/main/resources/kylin-defaults.properties | 2 +-
examples/test_case_data/sandbox/kylin.properties | 4 ++--
.../main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java | 4 ----
.../java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java | 3 ---
4 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index 00f433d..8903487 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -331,7 +331,7 @@ kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.executor.memory=4G
kylin.engine.spark-conf.spark.executor.instances=40
kylin.engine.spark-conf.spark.driver.cores=1
-kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024
+kylin.engine.spark-conf.spark.executor.memoryOverhead=1024
kylin.engine.spark-conf.spark.shuffle.service.enabled=true
kylin.engine.spark-conf.spark.eventLog.enabled=true
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 28b61cf..40ae6e1 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -196,8 +196,8 @@ kylin.engine.spark.rdd-partition-cut-mb=100
kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
kylin.engine.spark-conf.spark.master=yarn
#kylin.engine.spark-conf.spark.submit.deployMode=cluster
-kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=384
-kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=256
+kylin.engine.spark-conf.spark.executor.memoryOverhead=384
+kylin.engine.spark-conf.spark.driver.memoryOverhead=256
kylin.engine.spark-conf.spark.executor.memory=768M
kylin.engine.spark-conf.spark.executor.cores=1
kylin.engine.spark-conf.spark.executor.instances=1
diff --git a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java
index 96de267..4e0be04 100644
--- a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java
+++ b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java
@@ -20,11 +20,7 @@ package org.apache.kylin.engine.spark.job;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.model.Segments;
public class JobStepFactory {
diff --git a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java
index 14a424f..f788d15 100644
--- a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java
+++ b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java
@@ -35,10 +35,7 @@ import org.junit.Test;
import org.spark_project.guava.collect.Sets;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
public class JobStepFactoryTest extends LocalWithSparkSessionTest {
private KylinConfig config;