You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2016/07/23 00:00:06 UTC
drill git commit: DRILL-4743: Allow new options to control filter
selectivity (min/max bounds)
Repository: drill
Updated Branches:
refs/heads/master 04964bbf8 -> 4dac10347
DRILL-4743: Allow new options to control filter selectivity (min/max bounds)
Addressed review comments 2
Addressed review comments 3
close apache/drill#534
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4dac1034
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4dac1034
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4dac1034
Branch: refs/heads/master
Commit: 4dac1034739fb3ae563a76e7298344282650052e
Parents: 04964bb
Author: Gautam Parai <gp...@maprtech.com>
Authored: Tue Jun 21 12:30:02 2016 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Fri Jul 22 16:46:52 2016 -0700
----------------------------------------------------------------------
.../drill/exec/compile/QueryClassLoader.java | 4 +-
.../exec/planner/common/DrillFilterRelBase.java | 25 +++++++
.../exec/planner/cost/DrillRelMdRowCount.java | 6 ++
.../exec/planner/physical/PlannerSettings.java | 19 +++++
.../rpc/user/InboundImpersonationManager.java | 5 +-
.../server/options/FallbackOptionManager.java | 2 +-
.../exec/server/options/OptionValidator.java | 4 +-
.../server/options/SystemOptionManager.java | 4 +-
.../exec/server/options/TypeValidators.java | 77 ++++++++++++++++----
.../drill/exec/testing/ExecutionControls.java | 2 +-
.../java/org/apache/drill/PlanTestBase.java | 39 ++++++++++
.../java/org/apache/drill/TestSelectivity.java | 70 ++++++++++++++++++
.../TestInboundImpersonationPrivileges.java | 2 +-
.../exec/testing/ControlsInjectionUtil.java | 2 +-
14 files changed, 237 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
index 3df8f84..da03802 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
@@ -44,8 +44,8 @@ public class QueryClassLoader extends URLClassLoader {
public static final String JAVA_COMPILER_OPTION = "exec.java_compiler";
public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION, CompilerPolicy.DEFAULT.toString()) {
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
try {
CompilerPolicy.valueOf(v.string_val.toUpperCase());
} catch (IllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
index c52dede..45ff539 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
@@ -44,6 +44,8 @@ import java.util.List;
public abstract class DrillFilterRelBase extends Filter implements DrillRelNode {
private final int numConjuncts;
private final List<RexNode> conjunctions;
+ private final double filterMinSelectivityEstimateFactor;
+ private final double filterMaxSelectivityEstimateFactor;
protected DrillFilterRelBase(Convention convention, RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) {
super(cluster, traits, child, condition);
@@ -55,6 +57,10 @@ public abstract class DrillFilterRelBase extends Filter implements DrillRelNode
numConjuncts = conjunctions.size();
// assert numConjuncts >= 1;
+ filterMinSelectivityEstimateFactor = PrelUtil.
+ getPlannerSettings(cluster.getPlanner()).getFilterMinSelectivityEstimateFactor();
+ filterMaxSelectivityEstimateFactor = PrelUtil.
+ getPlannerSettings(cluster.getPlanner()).getFilterMaxSelectivityEstimateFactor();
}
@Override
@@ -90,4 +96,23 @@ public abstract class DrillFilterRelBase extends Filter implements DrillRelNode
return compNum * DrillCostBase.COMPARE_CPU_COST;
}
+ @Override
+ public double getRows() {
+ // override Calcite's default selectivity estimate - cap lower/upper bounds on the
+ // selectivity estimate in order to get desired parallelism
+ double selectivity = RelMetadataQuery.getSelectivity(getInput(), condition);
+ if (!condition.isAlwaysFalse()) {
+ // Cap selectivity at filterMinSelectivityEstimateFactor unless it is always FALSE
+ if (selectivity < filterMinSelectivityEstimateFactor) {
+ selectivity = filterMinSelectivityEstimateFactor;
+ }
+ }
+ if (!condition.isAlwaysTrue()) {
+ // Cap selectivity at filterMaxSelectivityEstimateFactor unless it is always TRUE
+ if (selectivity > filterMaxSelectivityEstimateFactor) {
+ selectivity = filterMaxSelectivityEstimateFactor;
+ }
+ }
+ return selectivity*RelMetadataQuery.getRowCount(getInput());
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdRowCount.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdRowCount.java
index b3c8834..25445e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdRowCount.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdRowCount.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.planner.cost;
import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdRowCount;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -39,4 +40,9 @@ public class DrillRelMdRowCount extends RelMdRowCount{
return super.getRowCount(rel);
}
}
+
+ @Override
+ public Double getRowCount(Filter rel) {
+ return rel.getRows();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index ff36d47..2a23d8b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -22,11 +22,15 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValidator;
+import org.apache.drill.exec.server.options.TypeValidators;
import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
import org.apache.drill.exec.server.options.TypeValidators.LongValidator;
+import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator;
import org.apache.drill.exec.server.options.TypeValidators.PositiveLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator;
import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator;
+import org.apache.drill.exec.server.options.TypeValidators.MinRangeDoubleValidator;
+import org.apache.drill.exec.server.options.TypeValidators.MaxRangeDoubleValidator;
import org.apache.calcite.plan.Context;
public class PlannerSettings implements Context{
@@ -81,6 +85,13 @@ public class PlannerSettings implements Context{
new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */,
Integer.MAX_VALUE, DEFAULT_IDENTIFIER_MAX_LENGTH);
+ public static final DoubleValidator FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR =
+ new MinRangeDoubleValidator("planner.filter.min_selectivity_estimate_factor",
+ 0.0, 1.0, 0.0d, "planner.filter.max_selectivity_estimate_factor");
+ public static final DoubleValidator FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR =
+ new MaxRangeDoubleValidator("planner.filter.max_selectivity_estimate_factor",
+ 0.0, 1.0, 1.0d, "planner.filter.min_selectivity_estimate_factor");
+
public static final String TYPE_INFERENCE_KEY = "planner.enable_type_inference";
public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY, true);
@@ -212,6 +223,14 @@ public class PlannerSettings implements Context{
return INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES;
}
+ public double getFilterMinSelectivityEstimateFactor() {
+ return options.getOption(FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR);
+ }
+
+ public double getFilterMaxSelectivityEstimateFactor(){
+ return options.getOption(FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR);
+ }
+
public boolean isTypeInferenceEnabled() {
return options.getOption(TYPE_INFERENCE);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java
index 06c3fe2..bd7c779 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.TypeValidators;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -90,8 +91,8 @@ public class InboundImpersonationManager {
}
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
final List<ImpersonationPolicy> policies;
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java
index 25ba0ad..f12b0e5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java
@@ -110,7 +110,7 @@ public abstract class FallbackOptionManager extends BaseOptionManager {
public void setOption(OptionValue value) {
final OptionValidator validator = SystemOptionManager.getValidator(value.name);
- validator.validate(value); // validate the option
+ validator.validate(value, this); // validate the option
// fallback if unable to set locally
if (!setLocalOption(value)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
index 3b43f9a..db42603 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
@@ -79,7 +79,9 @@ public abstract class OptionValidator {
* Validates the option value.
*
* @param value the value to validate
+ * @param manager the manager for accessing validation dependencies (options)
* @throws UserException message to describe error with value, including range or list of expected values
*/
- public abstract void validate(OptionValue value);
+ public abstract void validate(OptionValue value, OptionManager manager);
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 8584907..cfa3c40 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -83,6 +83,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
PlannerSettings.HEP_OPT,
PlannerSettings.PLANNER_MEMORY_LIMIT,
PlannerSettings.HEP_PARTITION_PRUNING,
+ PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR,
+ PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR,
PlannerSettings.TYPE_INFERENCE,
ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
@@ -251,7 +253,7 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
final String name = value.name.toLowerCase();
final OptionValidator validator = getValidator(name);
- validator.validate(value); // validate the option
+ validator.validate(value, this); // validate the option
if (options.get(name) == null && value.equals(validator.getDefault())) {
return; // if the option is not overridden, ignore setting option to default
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index ced448c..d015040 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -28,7 +28,6 @@ import static com.google.common.base.Preconditions.checkArgument;
public class TypeValidators {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeValidators.class);
-
public static class PositiveLongValidator extends LongValidator {
private final long max;
@@ -38,8 +37,8 @@ public class TypeValidators {
}
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
if (v.num_val > max || v.num_val < 1) {
throw UserException.validationError()
.message(String.format("Option %s must be between %d and %d.", getOptionName(), 1, max))
@@ -55,8 +54,8 @@ public class TypeValidators {
}
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
if (!isPowerOfTwo(v.num_val)) {
throw UserException.validationError()
.message(String.format("Option %s must be a power of two.", getOptionName()))
@@ -80,8 +79,8 @@ public class TypeValidators {
}
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
if (v.float_val > max || v.float_val < min) {
throw UserException.validationError()
.message(String.format("Option %s must be between %f and %f.", getOptionName(), min, max))
@@ -90,6 +89,56 @@ public class TypeValidators {
}
}
+ public static class MinRangeDoubleValidator extends RangeDoubleValidator {
+ private final double min;
+ private final double max;
+ private final String maxValidatorName;
+
+ public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) {
+ super(name, min, max, def);
+ this.min = min;
+ this.max = max;
+ this.maxValidatorName = maxValidatorName;
+ }
+
+ @Override
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
+ OptionValue maxValue = manager.getOption(maxValidatorName);
+ if (v.float_val > maxValue.float_val) {
+ throw UserException.validationError()
+ .message(String.format("Option %s must be less than or equal to Option %s",
+ getOptionName(), maxValidatorName))
+ .build(logger);
+ }
+ }
+ }
+
+ public static class MaxRangeDoubleValidator extends RangeDoubleValidator {
+ private final double min;
+ private final double max;
+ private final String minValidatorName;
+
+ public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) {
+ super(name, min, max, def);
+ this.min = min;
+ this.max = max;
+ this.minValidatorName = minValidatorName;
+ }
+
+ @Override
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
+ OptionValue minValue = manager.getOption(minValidatorName);
+ if (v.float_val < minValue.float_val) {
+ throw UserException.validationError()
+ .message(String.format("Option %s must be greater than or equal to Option %s",
+ getOptionName(), minValidatorName))
+ .build(logger);
+ }
+ }
+ }
+
public static class BooleanValidator extends TypeValidator {
public BooleanValidator(String name, boolean def) {
super(name, Kind.BOOLEAN, OptionValue.createBoolean(OptionType.SYSTEM, name, def));
@@ -125,8 +174,8 @@ public class TypeValidators {
}
@Override
- public void validate(OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
if (v.num_val > max || v.num_val < min) {
throw UserException.validationError()
.message(String.format("Option %s must be between %d and %d.", getOptionName(), min, max))
@@ -141,13 +190,13 @@ public class TypeValidators {
}
@Override
- public void validate(OptionValue v) {
+ public void validate(final OptionValue v, final OptionManager manager) {
if (v.type != OptionType.SYSTEM) {
throw UserException.validationError()
.message("Admin related settings can only be set at SYSTEM level scope. Given scope '%s'.", v.type)
.build(logger);
}
- super.validate(v);
+ super.validate(v, manager);
}
}
@@ -165,8 +214,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v) {
- super.validate(v);
+ public void validate(final OptionValue v, final OptionManager manager) {
+ super.validate(v, manager);
if (!valuesSet.contains(v.string_val.toLowerCase())) {
throw UserException.validationError()
.message(String.format("Option %s must be one of: %s.", getOptionName(), valuesSet))
@@ -192,7 +241,7 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v) {
+ public void validate(final OptionValue v, final OptionManager manager) {
if (v.kind != kind) {
throw UserException.validationError()
.message(String.format("Option %s must be of type %s but you tried to set to %s.", getOptionName(),
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
index 9673394..619055c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
@@ -96,7 +96,7 @@ public final class ExecutionControls {
}
@Override
- public void validate(final OptionValue v) {
+ public void validate(final OptionValue v, final OptionManager manager) {
if (v.type != OptionType.SESSION) {
throw UserException.validationError()
.message("Controls can be set only at SESSION level.")
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
index bb5ff88..5d1cdbf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
@@ -101,6 +101,45 @@ public class PlanTestBase extends BaseTestQuery {
}
/**
+ * Runs an explain plan including attributes query and check for expected regex patterns
+ * (in optiq text format), also ensure excluded patterns are not found. Either list can
+ * be empty or null to skip that part of the check.
+ *
+ * See the convenience methods for passing a single string in either the
+ * excluded list, included list or both.
+ *
+ * @param query - an explain query, this method does not add it for you
+ * @param expectedPatterns - list of patterns that should appear in the plan
+ * @param excludedPatterns - list of patterns that should not appear in the plan
+ * @throws Exception - if an inclusion or exclusion check fails, or the
+ * planning process throws an exception
+ */
+ public static void testPlanWithAttributesMatchingPatterns(String query, String[] expectedPatterns,
+ String[] excludedPatterns)
+ throws Exception {
+ final String plan = getPlanInString("EXPLAIN PLAN INCLUDING ALL ATTRIBUTES for " +
+ QueryTestUtil.normalizeQuery(query), OPTIQ_FORMAT);
+
+ // Check and make sure all expected patterns are in the plan
+ if (expectedPatterns != null) {
+ for (final String s : expectedPatterns) {
+ final Pattern p = Pattern.compile(s);
+ final Matcher m = p.matcher(plan);
+ assertTrue(EXPECTED_NOT_FOUND + s, m.find());
+ }
+ }
+
+ // Check and make sure all excluded patterns are not in the plan
+ if (excludedPatterns != null) {
+ for (final String s : excludedPatterns) {
+ final Pattern p = Pattern.compile(s);
+ final Matcher m = p.matcher(plan);
+ assertFalse(UNEXPECTED_FOUND + s, m.find());
+ }
+ }
+ }
+
+ /**
* Runs an explain plan query and check for expected substring patterns (in optiq
* text format), also ensure excluded patterns are not found. Either list can
* be empty or null to skip that part of the check.
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java b/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
new file mode 100644
index 0000000..648bd90
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.apache.drill.common.util.TestTools;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestSelectivity extends BaseTestQuery {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSelectivity.class);
+
+ @Test
+ public void testFilterSelectivityOptions() throws Exception {
+
+ /* Tests to check setting options works as expected */
+ test(String.format("alter system set `planner.filter.min_selectivity_estimate_factor` = %f", 0.25));
+ test(String.format("alter system set `planner.filter.max_selectivity_estimate_factor` = %f", 0.75));
+
+ String query1 = String.format("alter session set `planner.filter.min_selectivity_estimate_factor` = %f", -0.5);
+ String errMsg1 = "Option planner.filter.min_selectivity_estimate_factor must be between 0.000000 and 1.000000";
+ BaseTestQuery.errorMsgTestHelper(query1, errMsg1);
+
+ String query2 = String.format("alter session set `planner.filter.min_selectivity_estimate_factor` = %f", 0.85);
+ String errMsg2 = "Option planner.filter.min_selectivity_estimate_factor must be less than or equal to"
+ + " Option planner.filter.max_selectivity_estimate_factor";
+ BaseTestQuery.errorMsgTestHelper(query2, errMsg2);
+
+ String query3 = String.format("alter session set `planner.filter.max_selectivity_estimate_factor` = %f", 1.5);
+ String errMsg3 = "Option planner.filter.max_selectivity_estimate_factor must be between 0.000000 and 1.000000";
+ BaseTestQuery.errorMsgTestHelper(query3, errMsg3);
+
+ String query4 = String.format("alter session set `planner.filter.max_selectivity_estimate_factor` = %f", 0.15);
+ String errMsg4 = "Option planner.filter.max_selectivity_estimate_factor must be greater than or equal to"
+ + " Option planner.filter.min_selectivity_estimate_factor";
+ BaseTestQuery.errorMsgTestHelper(query4, errMsg4);
+
+ test(String.format("alter session set `planner.filter.max_selectivity_estimate_factor` = %f", 1.0));
+ test(String.format("alter session set `planner.filter.min_selectivity_estimate_factor` = %f", 0.9));
+ /* End of tests to check setting options */
+
+ /* Capping the selectivity prevents underestimation of filtered rows */
+ String query = " select employee_id from cp.`employee.json` where employee_id < 10 and department_id > 5";
+
+ test(String.format("alter session set `planner.filter.min_selectivity_estimate_factor` = %f", 0.1));
+ final String[] expectedPlan1 = {"Filter\\(condition.*\\).*rowcount = 115.75,.*",
+ "Scan.*columns=\\[`employee_id`, `department_id`\\].*rowcount = 463.0.*"};
+ PlanTestBase.testPlanWithAttributesMatchingPatterns(query, expectedPlan1, new String[]{});
+
+ test(String.format("alter session set `planner.filter.min_selectivity_estimate_factor` = %f", 0.9));
+ final String[] expectedPlan2 = {"Filter\\(condition.*\\).*rowcount = 416.7,.*",
+ "Scan.*columns=\\[`employee_id`, `department_id`\\].*rowcount = 463.0.*"};
+ PlanTestBase.testPlanWithAttributesMatchingPatterns(query, expectedPlan2, new String[]{});
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
index e5a0148..0d5393e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
@@ -49,7 +49,7 @@ public class TestInboundImpersonationPrivileges extends BaseTestImpersonation {
ExecConstants.IMPERSONATION_POLICY_VALIDATOR.validate(
OptionValue.createString(OptionValue.OptionType.SYSTEM,
ExecConstants.IMPERSONATION_POLICIES_KEY,
- IMPERSONATION_POLICIES));
+ IMPERSONATION_POLICIES), null);
try {
return InboundImpersonationManager.hasImpersonationPrivileges(proxyName, targetName, IMPERSONATION_POLICIES);
} catch (final Exception e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4dac1034/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
index 3f6de15..5365773 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
@@ -75,7 +75,7 @@ public class ControlsInjectionUtil {
final OptionManager options = session.getOptions();
try {
- DRILLBIT_CONTROLS_VALIDATOR.validate(opValue);
+ DRILLBIT_CONTROLS_VALIDATOR.validate(opValue, null);
options.setOption(opValue);
} catch (final Exception e) {
fail("Could not set controls options: " + e.getMessage());