You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/07 03:42:00 UTC
[02/10] kylin git commit: KYLIN-1984: Disable compress in default
configuration
KYLIN-1984: Disable compress in default configuration
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99d0d753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99d0d753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99d0d753
Branch: refs/heads/1.5.x-HBase1.x
Commit: 99d0d7535ae307a3b34a8ffd23b5d2d50559eea5
Parents: 663820f
Author: Yiming Liu <li...@gmail.com>
Authored: Sun Sep 4 17:22:20 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Sep 4 21:15:48 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 4 ++--
build/conf/kylin_hive_conf.xml | 13 ++++++++++++-
build/conf/kylin_job_conf.xml | 15 ++++++++++++---
build/conf/kylin_job_conf_inmem.xml | 15 ++++++++++++---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 5 +++--
6 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5607336..c20488a 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -54,8 +54,8 @@ kylin.storage.cleanup.time.threshold=172800000
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin
-# Compression codec for htable, valid value [snappy, lzo, gzip, lz4]
-kylin.hbase.default.compression.codec=snappy
+# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
+kylin.hbase.default.compression.codec=none
# HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020
# Leave empty if hbase running on same cluster with hive and mapreduce
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index 3d6109b..30c4feb 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -39,17 +39,28 @@
<description>enable map-side join</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
+ -->
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 877e82f..96b806c 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf_inmem.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf_inmem.xml b/build/conf/kylin_job_conf_inmem.xml
index 1cd809d..fea2f68 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1390e24..f0c91da 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -615,7 +615,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getHbaseDefaultCompressionCodec() {
- return getOptional("kylin.hbase.default.compression.codec", "");
+ return getOptional("kylin.hbase.default.compression.codec", "none");
}
public String getHbaseDefaultEncoding() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index fe65598..9b487a7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -183,8 +183,9 @@ public class CubeHTableUtil {
cf.setCompressionType(Algorithm.LZ4);
break;
}
+ case "none":
default: {
- logger.info("hbase will not user any compression algorithm to compress data");
+ logger.info("hbase will not use any compression algorithm to compress data");
cf.setCompressionType(Algorithm.NONE);
}
}
@@ -194,7 +195,7 @@ public class CubeHTableUtil {
DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
cf.setDataBlockEncoding(encoding);
} catch (Exception e) {
- logger.info("hbase will not use any encoding");
+ logger.info("hbase will not use any encoding", e);
cf.setDataBlockEncoding(DataBlockEncoding.NONE);
}