You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/09/04 14:06:21 UTC
[1/2] kylin git commit: KYLIN-1984: Disable compress in default
configuration
Repository: kylin
Updated Branches:
refs/heads/1.5.4-rc1 0534fdfc7 -> d37f0cafe
KYLIN-1984: Disable compress in default configuration
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d37f0caf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d37f0caf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d37f0caf
Branch: refs/heads/1.5.4-rc1
Commit: d37f0cafe46618875717f68f75ddb5c3d5280008
Parents: bb72093
Author: Yiming Liu <li...@gmail.com>
Authored: Sun Sep 4 17:22:20 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Sep 4 22:05:56 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 4 ++--
build/conf/kylin_hive_conf.xml | 13 ++++++++++++-
build/conf/kylin_job_conf.xml | 15 ++++++++++++---
build/conf/kylin_job_conf_inmem.xml | 15 ++++++++++++---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 5 +++--
6 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5607336..c20488a 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -54,8 +54,8 @@ kylin.storage.cleanup.time.threshold=172800000
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin
-# Compression codec for htable, valid value [snappy, lzo, gzip, lz4]
-kylin.hbase.default.compression.codec=snappy
+# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
+kylin.hbase.default.compression.codec=none
# HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020
# Leave empty if hbase running on same cluster with hive and mapreduce
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index 3d6109b..30c4feb 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -39,17 +39,28 @@
<description>enable map-side join</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
+ -->
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 877e82f..96b806c 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/build/conf/kylin_job_conf_inmem.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf_inmem.xml b/build/conf/kylin_job_conf_inmem.xml
index 1cd809d..fea2f68 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1390e24..f0c91da 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -615,7 +615,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getHbaseDefaultCompressionCodec() {
- return getOptional("kylin.hbase.default.compression.codec", "");
+ return getOptional("kylin.hbase.default.compression.codec", "none");
}
public String getHbaseDefaultEncoding() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d37f0caf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index fe65598..9b487a7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -183,8 +183,9 @@ public class CubeHTableUtil {
cf.setCompressionType(Algorithm.LZ4);
break;
}
+ case "none":
default: {
- logger.info("hbase will not user any compression algorithm to compress data");
+ logger.info("hbase will not use any compression algorithm to compress data");
cf.setCompressionType(Algorithm.NONE);
}
}
@@ -194,7 +195,7 @@ public class CubeHTableUtil {
DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
cf.setDataBlockEncoding(encoding);
} catch (Exception e) {
- logger.info("hbase will not use any encoding");
+ logger.info("hbase will not use any encoding", e);
cf.setDataBlockEncoding(DataBlockEncoding.NONE);
}
[2/2] kylin git commit: KYLIN-1992 Clear ThreadLocal Contexts when
query failed before scaning HBase
Posted by ma...@apache.org.
KYLIN-1992 Clear ThreadLocal Contexts when query failed before scaning HBase
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb72093d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb72093d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb72093d
Branch: refs/heads/1.5.4-rc1
Commit: bb72093d84bdc7d8bc7f1329ab2cfff237a63ee9
Parents: 0534fdf
Author: kangkaisen <ka...@live.com>
Authored: Thu Sep 1 16:02:42 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Sep 4 22:05:56 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bb72093d/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 3acaeb8..df296cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -316,6 +316,8 @@ public class QueryService extends BasicService {
parameters.put(OLAPContext.PRM_USER_AUTHEN_INFO, userInfo);
parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
OLAPContext.setParameters(parameters);
+ // force clear the query context before a new query
+ OLAPContext.clearThreadLocalContexts();
return execute(correctedSql, sqlRequest);