You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/04/18 09:51:40 UTC
[1/3] kylin git commit: stop testing storage type 0 as it is no
longer used
Repository: kylin
Updated Branches:
refs/heads/yang-m1 2f8df5160 -> 3aad1fc60
stop testing storage type 0 as it is no longer used
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/372d4335
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/372d4335
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/372d4335
Branch: refs/heads/yang-m1
Commit: 372d4335a0ef6cf913e68beaf599528f43eeb37c
Parents: 2f8df51
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Apr 18 11:27:45 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Apr 18 15:49:04 2016 +0800
----------------------------------------------------------------------
.../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +-
.../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index bb59a05..9d82ac3 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -166,6 +166,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 0,
- "storage_type" : 0,
+ "storage_type" : 2,
"partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index e1d740a..3741b22 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -330,6 +330,6 @@
"auto_merge_time_ranges": null,
"retention_range": 0,
"engine_type": 2,
- "storage_type": 0,
+ "storage_type": 2,
"partition_date_start": 0
}
[2/3] kylin git commit: stop testing storage type 0 as it is no
longer used
Posted by ma...@apache.org.
stop testing storage type 0 as it is no longer used
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad09341c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad09341c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad09341c
Branch: refs/heads/yang-m1
Commit: ad09341ca038c61351517d894920c64632bd5f5b
Parents: 372d433
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Apr 18 13:56:35 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Apr 18 15:51:14 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/common/RowKeySplitterTest.java | 6 +++---
.../test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +-
.../test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java | 3 +--
.../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +-
4 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
index 1ddc745..b41f23b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
@@ -48,7 +48,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase {
RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20);
// base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0,-104,-106,-128, 11, 54, -105, 55,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+ byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
rowKeySplitter.split(input);
assertEquals(11, rowKeySplitter.getBufferSize());
@@ -61,9 +61,9 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase {
RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20);
// base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+ byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
rowKeySplitter.split(input);
- assertEquals(9, rowKeySplitter.getBufferSize());
+ assertEquals(10, rowKeySplitter.getBufferSize());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 0fe37f7..43498d3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -53,7 +53,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
- byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+ byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
rowKeyDecoder.decode(key);
List<String> values = rowKeyDecoder.getValues();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 1a6b4dd..ba71387 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -70,13 +70,12 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
byte[] encodedKey = rowKeyEncoder.encode(data);
assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
- byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, rowKeyEncoder.getHeaderLength());
+ byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length);
assertEquals(255, Bytes.toLong(cuboidId));
assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
-
@Test
public void testEncodeWithSlr() throws Exception {
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index 9d82ac3..33a7080 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -165,7 +165,7 @@
"status_need_notify" : [ ],
"auto_merge_time_ranges" : null,
"retention_range" : 0,
- "engine_type" : 0,
+ "engine_type" : 2,
"storage_type" : 2,
"partition_date_start" : 0
}
\ No newline at end of file
[3/3] kylin git commit: KYLIN-1591 enabling different block size
setting for small column families
Posted by ma...@apache.org.
KYLIN-1591 enabling different block size setting for small column families
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6
Branch: refs/heads/yang-m1
Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba
Parents: ad09341
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Apr 18 15:51:31 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++---
.../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +++++++++++++
.../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++++++---
4 files changed, 28 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 94d8f62..23f0fa7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getQueryScanFuzzyKeyMax() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+ return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
}
public int getHBaseKeyValueSize() {
@@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
}
- public String getHbaseDefaultBlockSize() {
- return getOptional("kylin.hbase.default.block.size", "4194304");
+ public int getHbaseDefaultBlockSize() {
+ return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576"));
+ }
+
+ public int getHbaseSmallFamilyBlockSize() {
+ return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
}
public boolean isHiveKeepFlatTable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
import java.util.Arrays;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]";
}
+ public boolean isMemoryHungry() {
+ for (HBaseColumnDesc hBaseColumnDesc : columns) {
+ for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+ if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 97900c1..befa866 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -153,7 +153,8 @@ public class GTScanRequest {
}
if (!doAggr) {//Skip reading this section if you're not profiling!
- int scanned = lookAndForget(result);
+ int scanned = result.getScannedRowCount();
+ lookAndForget(result);
return new EmptyGTScanner(scanned);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 67c8518..8e3c134 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -91,6 +91,12 @@ public class CubeHTableUtil {
String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase();
+ if (cfDesc.isMemoryHungry()) {
+ cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize());
+ } else {
+ cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize());
+ }
+
switch (hbaseDefaultCC) {
case "snappy": {
logger.info("hbase will use snappy to compress data");
@@ -119,9 +125,6 @@ public class CubeHTableUtil {
}
}
- int blockSize = Integer.valueOf(kylinConfig.getHbaseDefaultBlockSize());
- cf.setBlocksize(blockSize);
-
try {
String encodingStr = kylinConfig.getHbaseDefaultEncoding();
DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);