You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/25 13:53:05 UTC
kylin git commit: KYLIN-1205 Patch scan in HBaseResourceStore as well
Repository: kylin
Updated Branches:
refs/heads/2.0-rc cbc86c265 -> b95d7282b
KYLIN-1205 Patch scan in HBaseResourceStore as well
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b95d7282
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b95d7282
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b95d7282
Branch: refs/heads/2.0-rc
Commit: b95d7282babbf13b6d1b0f22760519db43e7a2a0
Parents: cbc86c2
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 25 20:52:48 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 25 20:52:48 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/storage/hbase/HBaseResourceStore.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b95d7282/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 6f638c4..511a7cf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -156,6 +156,7 @@ public class HBaseResourceStore extends ResourceStore {
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
+ tuneScanParameters(scan);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
List<RawResource> result = Lists.newArrayList();
@@ -175,6 +176,13 @@ public class HBaseResourceStore extends ResourceStore {
return result;
}
+ private void tuneScanParameters(Scan scan) {
+ // divide by 10 as some resource like dictionary or snapshot can be very large
+ scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10);
+ scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize());
+ scan.setCacheBlocks(true);
+ }
+
private InputStream getInputStream(String resPath, Result r) throws IOException {
if (r == null) {
return null;
@@ -212,7 +220,7 @@ public class HBaseResourceStore extends ResourceStore {
protected long getResourceTimestampImpl(String resPath) throws IOException {
return getTimestamp(getByScan(resPath, false, true));
}
-
+
@Override
protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();