You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/01/10 00:46:08 UTC
[12/42] kylin git commit: KYLIN-1205 Patch HBaseResourceStore as well
KYLIN-1205 Patch HBaseResourceStore as well
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c98901b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c98901b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c98901b
Branch: refs/heads/1.x-HBase1.1.3
Commit: 7c98901bf8acaec29504e87e210fdf5e94a4b271
Parents: 95ef74f
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 25 21:08:21 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 25 21:08:21 2015 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/HBaseResourceStore.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c98901b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
index ac14e7b..d1ff27a 100644
--- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -148,6 +148,7 @@ public class HBaseResourceStore extends ResourceStore {
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
+ tuneScanParameters(scan);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
List<RawResource> result = Lists.newArrayList();
@@ -167,6 +168,13 @@ public class HBaseResourceStore extends ResourceStore {
return result;
}
+ private void tuneScanParameters(Scan scan) {
+ // divide by 10 as some resource like dictionary or snapshot can be very large
+ scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10);
+ scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize());
+ scan.setCacheBlocks(true);
+ }
+
private InputStream getInputStream(String resPath, Result r) throws IOException {
if (r == null) {
return null;
@@ -204,7 +212,7 @@ public class HBaseResourceStore extends ResourceStore {
protected long getResourceTimestampImpl(String resPath) throws IOException {
return getTimestamp(getByScan(resPath, false, true));
}
-
+
@Override
protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();