You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/12/30 02:31:06 UTC
[14/42] kylin git commit: add a toggle for dumpping storage partitions
add a toggle for dumpping storage partitions
add more log
temp for dumpped partitions
minor fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3
Branch: refs/heads/sparkcubing-rebase
Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284
Parents: 596ab51
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:33 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 30 ++++++++++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 2 ++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
}
+ public static String getPartitionDumpDir() {
+ return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+ }
+
+ public static String getDumpedPartitionDir() {
+ return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+ }
+
public static int getQueryTimeout() {
String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT";
+ /**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+ public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = "DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+ /**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+ public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
// properties on statement may go with this "channel" too
/**
* set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
public Iterator<GTRecord> iterator() {
Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
if (storagePushDownLimit != Integer.MAX_VALUE) {
+ logger.info("Using SortedIteratorMergerWithLimit to merge partitions");
return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
} else {
+ logger.info("Using Iterators.concat to merge partitions");
return Iterators.concat(shardSubsets);
}
}