You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/06 15:05:34 UTC
[kylin] 02/02: KYLIN-3137 allowing config Spark storage level
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 995a7198e6d78f0b21bac108a368306ca8cc48fa
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Jun 6 09:59:47 2018 +0800
KYLIN-3137 allowing config Spark storage level
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b289891..b2d8e99 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1200,6 +1200,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "5000"));
}
+ public String getSparkStorageLevel() {
+ return getOptional("kylin.engine.spark.storage-level", "MEMORY_AND_DISK_SER");
+ }
+
public boolean isSparkSanityCheckEnabled() {
return Boolean.parseBoolean(getOptional("kylin.engine.spark.sanity-check-enabled", "false"));
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 80a23ac..d609edb 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -172,7 +172,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
allNormalMeasure = allNormalMeasure && needAggr[i];
}
logger.info("All measure are normal (agg on all cuboids) ? : " + allNormalMeasure);
- StorageLevel storageLevel = StorageLevel.MEMORY_AND_DISK_SER();
+ StorageLevel storageLevel = StorageLevel.fromString(envConfig.getSparkStorageLevel());
boolean isSequenceFile = JoinedFlatTable.SEQUENCEFILE.equalsIgnoreCase(envConfig.getFlatTableStorageFormat());
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.