You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:27:03 UTC
[21/50] incubator-kylin git commit: KYLIN-702 When Kylin generates
large number of small files in flat hive table
KYLIN-702 When Kylin generates large number of small files in flat hive table
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7cd0d7dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7cd0d7dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7cd0d7dc
Branch: refs/heads/streaming-localdict
Commit: 7cd0d7dc79010cc4f978e43b93a590c63692c0ee
Parents: a02eb7c
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Mon Apr 20 10:37:39 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Mon Apr 20 10:42:15 2015 +0800
----------------------------------------------------------------------
conf/kylin_job_conf.xml | 21 +++++++++++++++++---
.../test_case_data/sandbox/kylin_job_conf.xml | 19 ++++++++++++++++++
2 files changed, 37 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7cd0d7dc/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/conf/kylin_job_conf.xml b/conf/kylin_job_conf.xml
index 49a95ec..e02c3e4 100644
--- a/conf/kylin_job_conf.xml
+++ b/conf/kylin_job_conf.xml
@@ -58,8 +58,7 @@
<description>Block replication</description>
</property>
- <!--Properties for calculating cube by splits, with which each Mapper need more mem to hold a full cube segment -->
- <!--
+ <!--Properties for calculating cube by splits (in-mem), with which each Mapper need more mem to hold a full cube segment -->
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx3072m</value>
@@ -73,5 +72,21 @@
<value>3600000</value>
<description>Set task timeout to 1 hour</description>
</property>
- -->
+
+ <property>
+ <name>hive.merge.mapfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on mapper only job</description>
+ </property>
+ <property>
+ <name>hive.merge.mapredfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on map-reduce job</description>
+ </property>
+
+ <property>
+ <name>hive.merge.size.per.task</name>
+ <value>16000000</value>
+ <description>Size for the merged file: 16M</description>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7cd0d7dc/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index f62ce55..92cf745 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -57,4 +57,23 @@
<value>2</value>
<description>Block replication</description>
</property>
+
+
+ <property>
+ <name>hive.merge.mapfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on mapper only job</description>
+ </property>
+
+ <property>
+ <name>hive.merge.mapredfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on map-reduce job</description>
+ </property>
+
+ <property>
+ <name>hive.merge.size.per.task</name>
+ <value>16000000</value>
+ <description>Size for the merged file: 16M</description>
+ </property>
</configuration>
\ No newline at end of file