You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/11/15 13:06:36 UTC
kylin git commit: KYLIN-2193 parameterise
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD
Repository: kylin
Updated Branches:
refs/heads/master 2c2970feb -> e68b25796
KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e68b2579
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e68b2579
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e68b2579
Branch: refs/heads/master
Commit: e68b257968a03809e114f6871394c4b55b05562f
Parents: 2c2970f
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Nov 15 21:06:25 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 5 ++++-
.../storage/translate/DerivedFilterTranslator.java | 13 +++++++++----
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ee9f57c..d44ff96 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -486,6 +486,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
+ public int getDerivedInThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+ }
+
public int getBadQueryStackTraceDepth() {
return Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
}
@@ -634,7 +638,6 @@ abstract public class KylinConfigBase implements Serializable {
return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
}
-
public boolean isHiveRedistributeEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.job.hive.intermediatetable.redistribute.enabled", "true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
/**
* @author yangli9
- *
*/
public class DerivedFilterTranslator {
-
- private static final int IN_THRESHOLD = 5;
+
+ private static final Logger logger = LoggerFactory.getLogger(DerivedFilterTranslator.class);
public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
TupleFilter translated;
boolean loosened;
- if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+ if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+ logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}",//
+ satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+ );
translated = buildRangeFilter(hostCols, satisfyingHostRecords);
loosened = true;
} else {