You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/04/27 04:59:59 UTC
kylin git commit: KYLIN-1627 add backdoor toggle to dump binary cube
storage response for further analysis
Repository: kylin
Updated Branches:
refs/heads/mhb_test_1 [created] 7405e48fc
KYLIN-1627 add backdoor toggle to dump binary cube storage response for further analysis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7405e48f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7405e48f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7405e48f
Branch: refs/heads/mhb_test_1
Commit: 7405e48fcee98e29ffa319e6744eae9825e9ef61
Parents: aebef62
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Apr 27 10:58:51 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Apr 27 10:59:08 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 14 ++++++++++++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 20 +++++++++++++++++---
2 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7405e48f/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 0feff2d..4a4a2f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -42,6 +42,10 @@ public class BackdoorToggles {
return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL);
}
+ public static String getStorageResultDump() {
+ return getString(DEBUG_TOGGLE_STORAGE_RESULT_DUMP);
+ }
+
public static boolean getDisableCache() {
return getBoolean(DEBUG_TOGGLE_DISABLE_QUERY_CACHE);
}
@@ -153,4 +157,14 @@ public class BackdoorToggles {
}
*/
public final static String DEBUG_TOGGLE_QUERY_TIMEOUT = "DEBUG_TOGGLE_QUERY_TIMEOUT";
+
+ /**
+ * set DEBUG_TOGGLE_STORAGE_RESULT_DUMP="path_to_dump" to dump the result from cube storage
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_STORAGE_RESULT_DUMP": "/kylin/data/"
+ }
+ */
+ public final static String DEBUG_TOGGLE_STORAGE_RESULT_DUMP = "DEBUG_TOGGLE_STORAGE_RESULT_DUMP";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7405e48f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 99a72bb..5d17764 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.hbase.cube.v2;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
@@ -36,6 +37,7 @@ import java.util.zip.DataFormatException;
import javax.annotation.Nullable;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.HConnection;
@@ -125,11 +127,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
this.timeout = HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
this.timeout *= KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
-
+
if (BackdoorToggles.getQueryTimeout() != -1) {
this.timeout = BackdoorToggles.getQueryTimeout();
}
-
+
this.timeout *= 1.1;//allow for some delay
logger.info("Timeout for ExpectedSizeIterator is: " + this.timeout);
@@ -319,6 +321,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicInteger totalScannedCount = new AtomicInteger(0);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size() * shardNum);
+ final AtomicInteger atomicInteger = new AtomicInteger(0);
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
@@ -352,7 +355,17 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
abnormalFinish = true;
} else {
try {
- epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows())));
+ int order = atomicInteger.incrementAndGet();
+ byte[] compressed = HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows());
+ if (BackdoorToggles.getStorageResultDump() != null) {
+ try {
+ FileUtils.writeByteArrayToFile(new File(BackdoorToggles.getStorageResultDump() + "/" + order), compressed);
+ logger.info("write to file for part " + order + " finished");
+ } catch (Exception e) {
+ logger.error("failed to write for part " + order, e);
+ }
+ }
+ epResultItr.append(CompressionUtils.decompress(compressed));
} catch (IOException | DataFormatException e) {
throw new RuntimeException(logHeader + "Error when decompressing", e);
}
@@ -368,6 +381,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
return new EndpointResultsAsGTScanner(fullGTInfo, epResultItr, scanRequests.get(0).getColumns(), totalScannedCount.get());
+
}
private String getStatsString(Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result) {