You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/27 10:34:05 UTC
incubator-kylin git commit: KYLIN-942 add more backdoor toggles to
help profiling
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942-x 417faedf9 -> 1b3b08eb9
KYLIN-942 add more backdoor toggles to help profiling
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1b3b08eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1b3b08eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1b3b08eb
Branch: refs/heads/KYLIN-942-x
Commit: 1b3b08eb9b7f1cf4b35a97af15619a894bd47f46
Parents: 417faed
Author: honma <ho...@ebay.com>
Authored: Tue Oct 27 17:38:26 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Oct 27 17:38:26 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 29 ++++++++++++++++++++
.../kylin/storage/hbase/HBaseStorage.java | 23 ++++++++++++----
.../hbase/cube/v2/CubeSegmentScanner.java | 9 ++++--
3 files changed, 54 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b3b08eb/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 53e5864..bac7258 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -34,6 +34,14 @@ public class BackdoorToggles {
return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
}
+ public static String getHbaseCubeQueryVersion() {
+ return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION);
+ }
+
+ public static String getHbaseCubeQueryProtocol() {
+ return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL);
+ }
+
public static boolean getDisableFuzzyKey() {
return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
}
@@ -72,6 +80,27 @@ public class BackdoorToggles {
public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
/**
+ * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION=v1/v2 to control which version CubeStorageQuery to use
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION": "v1"
+ }
+ */
+ public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION";
+
+ /**
+ * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL=endpoint/scan to control how to visit hbase cube
+ * this param is only valid when DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION set to v2(bdefault)
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL": "scan"
+ }
+ */
+ public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL";
+
+ /**
* set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
*
example:(put it into request body)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b3b08eb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 53465d8..2e2000d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.hbase;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.engine.mr.IMROutput;
import org.apache.kylin.engine.mr.IMROutput2;
@@ -39,11 +40,15 @@ import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
import com.google.common.base.Preconditions;
-@SuppressWarnings("unused")//used by reflection
+@SuppressWarnings("unused")
+//used by reflection
public class HBaseStorage implements IStorage {
private final static boolean allowStorageLayerCache = true;
- private final static String defaultCubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
+
+ private final static String v2CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
+ private final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery";
+
private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery";
@Override
@@ -62,11 +67,19 @@ public class HBaseStorage implements IStorage {
return ret;
}
} else if (realization.getType() == RealizationType.CUBE) {
+
+ String cubeStorageQuery;
+ if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) {
+ cubeStorageQuery = v1CubeStorageQuery;
+ } else {
+ cubeStorageQuery = v2CubeStorageQuery;//by default use v2
+ }
+
ICachableStorageQuery ret;
try {
- ret = (ICachableStorageQuery) Class.forName(defaultCubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization);
+ ret = (ICachableStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization);
} catch (Exception e) {
- throw new RuntimeException("Failed to initialize storage query for " + defaultCubeStorageQuery, e);
+ throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
}
if (allowStorageLayerCache) {
@@ -78,7 +91,7 @@ public class HBaseStorage implements IStorage {
throw new IllegalArgumentException("Unknown realization type " + realization.getType());
}
}
-
+
private static IStorageQuery wrapWithCache(ICachableStorageQuery underlyingStorageEngine, IRealization realization) {
if (underlyingStorageEngine.isDynamic()) {
return new CacheFledgedDynamicQuery(underlyingStorageEngine, getPartitionCol(realization));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b3b08eb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 814a359..18ba6fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -11,6 +11,7 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -224,8 +225,12 @@ public class CubeSegmentScanner implements IGTScanner {
try {
- CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
- //CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
+ CubeHBaseRPC rpc;
+ if ("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) {
+ rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
+ } else {
+ rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);//default behavior
+ }
//change previous line to CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
//to debug locally