You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/04/18 09:51:42 UTC
[3/3] kylin git commit: KYLIN-1591 enabling different block size
setting for small column families
KYLIN-1591 enabling different block size setting for small column families
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6
Branch: refs/heads/yang-m1
Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba
Parents: ad09341
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Apr 18 15:51:31 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++---
.../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +++++++++++++
.../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++++++---
4 files changed, 28 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 94d8f62..23f0fa7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getQueryScanFuzzyKeyMax() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+ return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
}
public int getHBaseKeyValueSize() {
@@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
}
- public String getHbaseDefaultBlockSize() {
- return getOptional("kylin.hbase.default.block.size", "4194304");
+ public int getHbaseDefaultBlockSize() {
+ return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576"));
+ }
+
+ public int getHbaseSmallFamilyBlockSize() {
+ return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
}
public boolean isHiveKeepFlatTable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
import java.util.Arrays;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]";
}
+ public boolean isMemoryHungry() {
+ for (HBaseColumnDesc hBaseColumnDesc : columns) {
+ for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+ if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 97900c1..befa866 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -153,7 +153,8 @@ public class GTScanRequest {
}
if (!doAggr) {//Skip reading this section if you're not profiling!
- int scanned = lookAndForget(result);
+ int scanned = result.getScannedRowCount();
+ lookAndForget(result);
return new EmptyGTScanner(scanned);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 67c8518..8e3c134 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -91,6 +91,12 @@ public class CubeHTableUtil {
String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase();
+ if (cfDesc.isMemoryHungry()) {
+ cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize());
+ } else {
+ cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize());
+ }
+
switch (hbaseDefaultCC) {
case "snappy": {
logger.info("hbase will use snappy to compress data");
@@ -119,9 +125,6 @@ public class CubeHTableUtil {
}
}
- int blockSize = Integer.valueOf(kylinConfig.getHbaseDefaultBlockSize());
- cf.setBlocksize(blockSize);
-
try {
String encodingStr = kylinConfig.getHbaseDefaultEncoding();
DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);