You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/02/07 09:52:46 UTC
[kylin] branch master updated: KYLIN-4896 Optimize the process of
writing big resource files into HDFS.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 985ff83 KYLIN-4896 Optimize the process of writing big resource files into HDFS.
985ff83 is described below
commit 985ff834a2eefc536a3bb2e516ab36d8f5667893
Author: helenzeng0503 <li...@163.com>
AuthorDate: Fri Feb 5 09:40:30 2021 +0800
KYLIN-4896 Optimize the process of writing big resource files into HDFS.
---
.../main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index fc00ec2..230ee73 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -88,8 +88,10 @@ public class HBaseResourceStore extends PushdownResourceStore {
tableName = metadataUrl.getIdentifier();
createHTableIfNeeded(tableName);
- kvSizeLimit = Integer
+ int kvSizeLimitActual = Integer
.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
+ kvSizeLimit = kvSizeLimitActual > 10485760 ? kvSizeLimitActual : 10485760;
+ logger.debug("hbase.client.keyvalue.maxsize is {}, kvSizeLimit is set to {}", kvSizeLimitActual, kvSizeLimit);
}
protected Connection getConnection() throws IOException {
@@ -328,6 +330,7 @@ public class HBaseResourceStore extends PushdownResourceStore {
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
if (content.length > kvSizeLimit) {
+ logger.info("Length of content exceeds the limit of {} bytes, push down {} to HDFS", kvSizeLimit, resPath);
pushdown = writePushdown(resPath, ContentWriter.create(content));
content = BytesUtil.EMPTY_BYTE_ARRAY;
}