You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/27 00:35:05 UTC
[41/50] [abbrv] kylin git commit: KYLIN-1565: Read KV max size from
HBase config
KYLIN-1565: Read KV max size from HBase config
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e05cb600
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e05cb600
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e05cb600
Branch: refs/heads/1.5.x-CDH5.7
Commit: e05cb60061a43987f7648d3d04921b5a01bb5ac9
Parents: 0861bfb
Author: Yiming Liu <li...@gmail.com>
Authored: Fri Sep 16 11:51:11 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Sep 26 19:45:19 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 4 ----
.../kylin/storage/hbase/ITHBaseResourceStoreTest.java | 10 +++++-----
.../apache/kylin/storage/hbase/HBaseResourceStore.java | 2 +-
3 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 79ee084..ceb188e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -570,10 +570,6 @@ abstract public class KylinConfigBase implements Serializable {
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
}
- public int getHBaseKeyValueSize() {
- return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
- }
-
public String getDefaultIGTStorage() {
return getOptional("kylin.query.storage.default.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index bc5cdf1..5a2d5dd 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -57,12 +57,12 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
String path = "/cube/_test_large_cell.json";
String largeContent = "THIS_IS_A_LARGE_CELL";
StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
+ Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+ int origSize = Integer.parseInt(hconf.get("hbase.client.keyvalue.maxsize", "10485760"));
ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+ hconf.set("hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
store.deleteResource(path);
@@ -72,7 +72,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
assertEquals(content, t);
Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+
FileSystem fileSystem = FileSystem.get(hconf);
assertTrue(fileSystem.exists(redirectPath));
@@ -82,7 +82,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
store.deleteResource(path);
} finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ hconf.set("hbase.client.keyvalue.maxsize", "" + origSize);
store.deleteResource(path);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..ee4b7e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -364,7 +364,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
- int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
+ int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
content = BytesUtil.EMPTY_BYTE_ARRAY;