You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/25 08:30:04 UTC
incubator-kylin git commit: minor,
add configuration to run coprocessor locally only
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 249a500c3 -> 7389f89e1
minor, add configuration to run coprocessor locally only
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7389f89e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7389f89e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7389f89e
Branch: refs/heads/0.8
Commit: 7389f89e14ca741db448b89d221dd68e9e37a148
Parents: 249a500
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 25 14:29:49 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 25 14:29:49 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 27 ++----
.../kylin/common/debug/BackdoorToggles.java | 93 ++++++++++++++++++++
.../coprocessor/observer/ObserverEnabler.java | 8 +-
3 files changed, 104 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7389f89e/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index ea9adf9..7f9930e 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -40,8 +40,6 @@ public class KylinConfig {
public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
- public static final String PROP_SCAN_THRESHOLD = "PROP_SCAN_THRESHOLD";
-
public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
@@ -94,8 +92,6 @@ public class KylinConfig {
public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
-
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
public static final String KYLIN_JOB_CUBING_IN_MEM = "kylin.job.cubing.inMem";
@@ -354,10 +350,6 @@ public class KylinConfig {
System.setProperty(COPROCESSOR_LOCAL_JAR, path);
}
- public int getCoprocessorScanBitsThreshold() {
- return Integer.parseInt(getOptional(COPROCESSOR_SCAN_BITS_THRESHOLD, "32"));
- }
-
public double getDefaultHadoopJobReducerInputMB() {
return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
}
@@ -441,10 +433,14 @@ public class KylinConfig {
public int getDictionaryMaxCardinality() {
return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
}
-
+
public int getScanThreshold() {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
+
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
+ }
public Long getQueryDurationCacheThreshold() {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
@@ -458,10 +454,6 @@ public class KylinConfig {
return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
}
- public int getConcurrentScanThreadCount() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.thread.count", "40"));
- }
-
public boolean isQueryCacheEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
}
@@ -661,15 +653,6 @@ public class KylinConfig {
kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v);
}
- /**
- * return -1 if there is no setting
- *
- * @return
- */
- public int getPropScanThreshold() {
- return kylinConfig.getInt(PROP_SCAN_THRESHOLD, -1);
- }
-
public String getProperty(String key, String defaultValue) {
return kylinConfig.getString(key, defaultValue);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7389f89e/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
new file mode 100644
index 0000000..5ce76aa
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.debug;
+
+import java.util.Map;
+
+/**
+ */
+public class BackdoorToggles {
+
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
+
+ public static void setToggles(Map<String, String> toggles) {
+ _backdoorToggles.set(toggles);
+ }
+
+ public static String getObserverBehavior() {
+ return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ }
+
+ public static boolean getDisableFuzzyKey() {
+ return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
+ }
+
+ public static boolean getRunLocalCoprocessor() {
+ return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
+ }
+
+ private static String getString(String key) {
+ Map<String, String> toggles = _backdoorToggles.get();
+ if (toggles == null) {
+ return null;
+ } else {
+ return toggles.get(key);
+ }
+ }
+
+ private static boolean getBoolean(String key) {
+ return "true".equals(getString(key));
+ }
+
+ public static void cleanToggles() {
+ _backdoorToggles.remove();
+ }
+
+ /**
+ * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
+ *
+ *
+ *
+ example:
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ }
+ */
+ public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
+
+ /**
+ * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ *
+ example:
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ }
+ */
+ public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+
+ /**
+ * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
+ *
+ example:
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+ }
+ */
+ public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7389f89e/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index b8b6343..9b3753d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -37,6 +37,9 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -54,7 +57,6 @@ public class ObserverEnabler {
private static final Logger logger = LoggerFactory.getLogger(ObserverEnabler.class);
static final String FORCE_COPROCESSOR = "forceObserver";
- static final boolean DEBUG_LOCAL_COPROCESSOR = false;
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
@@ -68,8 +70,10 @@ public class ObserverEnabler {
CoprocessorFilter filter = CoprocessorFilter.fromFilter(segment, tupleFiler, FilterDecorator.FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT);
CoprocessorProjector projector = CoprocessorProjector.makeForObserver(segment, cuboid, groupBy);
ObserverAggregators aggrs = ObserverAggregators.fromValueDecoders(rowValueDecoders);
+
+ boolean localCoprocessor = KylinConfig.getInstanceFromEnv().getQueryRunLocalCoprocessor() || BackdoorToggles.getRunLocalCoprocessor();
- if (DEBUG_LOCAL_COPROCESSOR) {
+ if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner);
return new ResultScannerAdapter(aggrScanner);