You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/01/14 10:07:14 UTC
[kylin] branch master updated: KYLIN-4322: set storage.hbase.endpoint-compress-result default value … (#1033)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new f41c6c8 KYLIN-4322: set storage.hbase.endpoint-compress-result default value … (#1033)
f41c6c8 is described below
commit f41c6c8198e5cad295e9212c6a0047d83bd54ae2
Author: Kang <zh...@xiaomi.com>
AuthorDate: Tue Jan 14 18:07:04 2020 +0800
KYLIN-4322: set storage.hbase.endpoint-compress-result default value … (#1033)
* KYLIN-4322: set storage.hbase.endpoint-compress-result default value false
* KYLIN-4322: update UT
---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../coprocessor/endpoint/CubeVisitServiceTest.java | 24 ++++++++++++++++------
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 44629e6..d9561fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1339,7 +1339,7 @@ public abstract class KylinConfigBase implements Serializable {
}
public boolean getCompressionResult() {
- return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", TRUE));
+ return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", FALSE));
}
public int getHBaseMaxConnectionThreads() {
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
index b3e5a93..be822f2 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
+import java.util.zip.DataFormatException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -171,6 +172,20 @@ public class CubeVisitServiceTest extends LocalFileMetadataTestCase {
service.start(env);
}
+ private byte[] getResult(CubeVisitProtos.CubeVisitResponse result) throws IOException, DataFormatException {
+ boolean compressionResult = KylinConfig.getInstanceFromEnv().getCompressionResult();
+ byte[] rawData = HBaseZeroCopyByteString.zeroCopyGetBytes(result.getCompressedRows());
+ byte[] resultData = null;
+ if (compressionResult) {
+ resultData = CompressionUtils.decompress(rawData);
+ } else {
+ resultData = rawData;
+ }
+
+ return resultData;
+ }
+
+
@Test
public void testVisitCube() throws Exception {
RawScan rawScan = mockFullScan(gtInfo, getTestConfig());
@@ -192,8 +207,7 @@ public class CubeVisitServiceTest extends LocalFileMetadataTestCase {
Assert.assertEquals(dateList.size() * userList.size(), stats.getScannedRowCount());
try {
- byte[] rawData = CompressionUtils
- .decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getCompressedRows()));
+ byte[] rawData = getResult(result);
PartitionResultIterator iterator = new PartitionResultIterator(rawData, gtInfo, setOf(0, 1, 2, 3));
int nReturn = 0;
while (iterator.hasNext()) {
@@ -226,8 +240,7 @@ public class CubeVisitServiceTest extends LocalFileMetadataTestCase {
@Override
public void run(CubeVisitProtos.CubeVisitResponse result) {
try {
- byte[] rawData = CompressionUtils
- .decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getCompressedRows()));
+ byte[] rawData = getResult(result);
PartitionResultIterator iterator = new PartitionResultIterator(rawData, gtInfo, setOf(1, 3));
Map<String, BigDecimal> actRet = Maps.newHashMap();
while (iterator.hasNext()) {
@@ -285,8 +298,7 @@ public class CubeVisitServiceTest extends LocalFileMetadataTestCase {
@Override
public void run(CubeVisitProtos.CubeVisitResponse result) {
try {
- byte[] rawData = CompressionUtils
- .decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getCompressedRows()));
+ byte[] rawData = getResult(result);
PartitionResultIterator iterator = new PartitionResultIterator(rawData, gtInfo, setOf(2, 3));
Map<BigDecimal, BigDecimal> actRet = Maps.newHashMap();
while (iterator.hasNext()) {