You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/24 08:38:08 UTC
[5/7] kylin git commit: minor,
adjust out-of-box kylin.properties for sparking cubing
minor, adjust out-of-box kylin.properties for sparking cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ff78bce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ff78bce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ff78bce
Branch: refs/heads/master-hbase0.98
Commit: 9ff78bce72a56b63dc9e1c8f0b4cd59d0c01b011
Parents: d59b8ae
Author: Li Yang <li...@apache.org>
Authored: Fri Feb 24 16:15:52 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Feb 24 16:20:33 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 17 ++++++++---------
.../apache/kylin/engine/spark/SparkExecutable.java | 4 ++--
webapp/app/js/model/cubeConfig.js | 2 +-
3 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index d7ba0b3..46fe86b 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -203,8 +203,9 @@ kylin.security.saml.context-path=/kylin
### Spark Engine Configs ###
-# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
-#kylin.env.hadoop-conf-dir=/etc/hive/conf
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" to run spark-submit
+# This must contain site xmls of core, yarn, hive, and hbase in one folder
+#kylin.env.hadoop-conf-dir=/etc/hadoop/conf
# Estimate the RDD partition numbers
kylin.engine.spark.rdd-partition-cut-mb=10
@@ -215,19 +216,17 @@ kylin.engine.spark.min-partition=1
# Max partition numbers of rdd
kylin.engine.spark.max-partition=5000
-### Spark conf (default is in spark/conf/spark-defaults.conf)
+## Spark conf (default is in spark/conf/spark-defaults.conf)
kylin.engine.spark-conf.spark.master=yarn
kylin.engine.spark-conf.spark.submit.deployMode=cluster
-kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024
-kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384
kylin.engine.spark-conf.spark.yarn.queue=default
-kylin.engine.spark-conf.spark.executor.memory=4G
-kylin.engine.spark-conf.spark.executor.cores=4
-kylin.engine.spark-conf.spark.executor.instances=8
-kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
+kylin.engine.spark-conf.spark.executor.memory=1G
+kylin.engine.spark-conf.spark.executor.cores=2
+kylin.engine.spark-conf.spark.executor.instances=1
kylin.engine.spark-conf.spark.eventLog.enabled=true
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+
## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index cf7438c..5ad21cf 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -71,7 +71,7 @@ public class SparkExecutable extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final KylinConfig config = context.getConfig();
- if (config.getSparkHome() == null) {
+ if (KylinConfig.getSparkHome() == null) {
throw new NullPointerException();
}
if (config.getKylinJobJarPath() == null) {
@@ -117,7 +117,7 @@ public class SparkExecutable extends AbstractExecutable {
stringBuilder.append("--files %s --jars %s %s %s");
try {
- String cmd = String.format(stringBuilder.toString(), hadoopConf, config.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
+ String cmd = String.format(stringBuilder.toString(), hadoopConf, KylinConfig.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
logger.info("cmd:" + cmd);
final StringBuilder output = new StringBuilder();
CliCommandExecutor exec = new CliCommandExecutor();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 47d0f46..eec3f10 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -25,7 +25,7 @@ KylinApp.constant('cubeConfig', {
cubePartitionTypes: ['APPEND'],
engineType:[
{name:'MapReduce',value: 2},
- {name:'Spark(Beta)',value: 4}
+ {name:'Spark (Beta)',value: 4}
],
joinTypes: [
{name: 'Left', value: 'left'},