You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/25 08:04:55 UTC
[2/2] incubator-kylin git commit: minor,
add configuration to disable run remote coprocessor
minor, add configuration to disable run remote coprocessor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/faeb8a96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/faeb8a96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/faeb8a96
Branch: refs/heads/0.7-staging
Commit: faeb8a9689e39f983599f36e5681fff040345e19
Parents: 613106c
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 25 14:03:58 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 25 14:04:42 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 27 +++--------------
.../kylin/common/debug/BackdoorToggles.java | 32 ++++++++++++++++++--
.../kylin/storage/hbase/HBaseKeyRange.java | 11 +++----
.../coprocessor/observer/ObserverEnabler.java | 15 +++++----
4 files changed, 48 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index dde3ba5..5ae18a6 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -40,8 +40,6 @@ public class KylinConfig {
public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
- public static final String PROP_SCAN_THRESHOLD = "PROP_SCAN_THRESHOLD";
-
public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
@@ -92,8 +90,6 @@ public class KylinConfig {
public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
-
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
@@ -345,10 +341,6 @@ public class KylinConfig {
System.setProperty(COPROCESSOR_LOCAL_JAR, path);
}
- public int getCoprocessorScanBitsThreshold() {
- return Integer.parseInt(getOptional(COPROCESSOR_SCAN_BITS_THRESHOLD, "32"));
- }
-
public double getDefaultHadoopJobReducerInputMB() {
return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
}
@@ -428,10 +420,14 @@ public class KylinConfig {
public int getDictionaryMaxCardinality() {
return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
}
-
+
public int getScanThreshold() {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
+
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
+ }
public Long getQueryDurationCacheThreshold() {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
@@ -445,10 +441,6 @@ public class KylinConfig {
return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
}
- public int getConcurrentScanThreadCount() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.thread.count", "40"));
- }
-
public boolean isQueryCacheEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
}
@@ -637,15 +629,6 @@ public class KylinConfig {
kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v);
}
- /**
- * return -1 if there is no setting
- *
- * @return
- */
- public int getPropScanThreshold() {
- return kylinConfig.getInt(PROP_SCAN_THRESHOLD, -1);
- }
-
public String getProperty(String key, String defaultValue) {
return kylinConfig.getString(key, defaultValue);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 315543f..5760798 100644
--- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -6,13 +6,25 @@ import java.util.Map;
*/
public class BackdoorToggles {
- private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal();
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
public static void setToggles(Map<String, String> toggles) {
_backdoorToggles.set(toggles);
}
- public static String getToggle(String key) {
+ public static String getObserverBehavior() {
+ return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ }
+
+ public static boolean getDisableFuzzyKey() {
+ return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
+ }
+
+ public static boolean getRunLocalCoprocessor() {
+ return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
+ }
+
+ private static String getString(String key) {
Map<String, String> toggles = _backdoorToggles.get();
if (toggles == null) {
return null;
@@ -20,6 +32,10 @@ public class BackdoorToggles {
return toggles.get(key);
}
}
+
+ private static boolean getBoolean(String key) {
+ return "true".equals(getString(key));
+ }
public static void cleanToggles() {
_backdoorToggles.remove();
@@ -50,4 +66,16 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+
+ /**
+ * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index 278c50c..9b05dc7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -168,13 +168,10 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
- //debug/profiling purpose
- String toggle;
- if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_DISABLE_FUZZY_KEY)) != null) {
- if (Boolean.valueOf(toggle)) {
- logger.info("The execution of this query will not use fuzzy key");
- return result;
- }
+ // debug/profiling purpose
+ if (BackdoorToggles.getDisableFuzzyKey()) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
}
FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index 6acc40e..84688a1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -20,10 +20,12 @@ package org.apache.kylin.storage.hbase.coprocessor.observer;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
@@ -53,7 +55,6 @@ public class ObserverEnabler {
private static final Logger logger = LoggerFactory.getLogger(ObserverEnabler.class);
static final String FORCE_COPROCESSOR = "forceObserver";
- static final boolean DEBUG_LOCAL_COPROCESSOR = false;
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
@@ -67,17 +68,19 @@ public class ObserverEnabler {
CoprocessorFilter filter = CoprocessorFilter.fromFilter(segment, tupleFiler);
CoprocessorProjector projector = CoprocessorProjector.makeForObserver(segment, cuboid, groupBy);
ObserverAggregators aggrs = ObserverAggregators.fromValueDecoders(rowValueDecoders);
+
+ boolean localCoprocessor = KylinConfig.getInstanceFromEnv().getQueryRunLocalCoprocessor() || BackdoorToggles.getRunLocalCoprocessor();
- if (DEBUG_LOCAL_COPROCESSOR) {
+ if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
- //debug/profiling purpose
- String toggle;
- if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_OBSERVER_BEHAVIOR)) == null) {
- toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString();//default behavior
+ // debug/profiling purpose
+ String toggle = BackdoorToggles.getObserverBehavior();
+ if (toggle == null) {
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}