You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by sa...@apache.org on 2016/12/07 06:41:47 UTC
[4/4] incubator-impala git commit: IMPALA-4592: Improve error msg for
non-deterministic predicates.
IMPALA-4592: Improve error msg for non-deterministic predicates.
Impala cannot correctly evaluate or assign some non-deterministic
predicates. This patch improves the error message shown when
trying to evaluate such unsupported predicates for the purpose
of partition pruning.
Change-Id: I94765f62bde94f4faa7fc5c26d928099ca1496d1
Reviewed-on: http://gerrit.cloudera.org:8080/5386
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6098ac71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6098ac71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6098ac71
Branch: refs/heads/master
Commit: 6098ac7162742c11350de708188ce6c3f7ce11a7
Parents: 5188f87
Author: Alex Behm <al...@cloudera.com>
Authored: Tue Dec 6 14:30:43 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Dec 7 06:27:51 2016 +0000
----------------------------------------------------------------------
.../org/apache/impala/analysis/PartitionSet.java | 5 +++--
.../impala/planner/HdfsPartitionFilter.java | 18 +++++++++++-------
.../impala/planner/HdfsPartitionPruner.java | 6 +++---
.../queries/PlannerTest/hdfs.test | 11 +++++++++++
4 files changed, 28 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
index 6de2d93..3ba2ad2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
@@ -25,7 +25,7 @@ import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.Table;
import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Reference;
import org.apache.impala.planner.HdfsPartitionPruner;
import org.apache.impala.thrift.TPartitionKeyValue;
@@ -87,7 +87,8 @@ public class PartitionSet extends PartitionSpecBase {
try {
HdfsPartitionPruner pruner = new HdfsPartitionPruner(desc);
partitions_ = pruner.prunePartitions(analyzer, transformedConjuncts, true);
- } catch (InternalException e) {
+ } catch (ImpalaException e) {
+ if (e instanceof AnalysisException) throw (AnalysisException) e;
throw new AnalysisException("Partition expr evaluation failed in the backend.", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
index 7368358..3c0fb15 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
@@ -21,9 +21,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.ExprSubstitutionMap;
@@ -33,10 +30,14 @@ import org.apache.impala.analysis.SlotRef;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.NotImplementedException;
import org.apache.impala.service.FeSupport;
import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TResultRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -83,7 +84,7 @@ public class HdfsPartitionFilter {
* that pass the filter.
*/
public HashSet<Long> getMatchingPartitionIds(ArrayList<HdfsPartition> partitions,
- Analyzer analyzer) throws InternalException {
+ Analyzer analyzer) throws ImpalaException {
HashSet<Long> result = new HashSet<Long>();
// List of predicates to evaluate
ArrayList<Expr> predicates = new ArrayList<Expr>(partitions.size());
@@ -110,7 +111,7 @@ public class HdfsPartitionFilter {
* for the partition cols with the respective partition-key values.
*/
private Expr buildPartitionPredicate(HdfsPartition partition, Analyzer analyzer)
- throws InternalException {
+ throws ImpalaException {
// construct smap
ExprSubstitutionMap sMap = new ExprSubstitutionMap();
for (int i = 0; i < refdKeys_.size(); ++i) {
@@ -123,7 +124,10 @@ public class HdfsPartitionFilter {
LOG.trace("buildPartitionPredicate: " + literalPredicate.toSql() + " " +
literalPredicate.debugString());
}
- Preconditions.checkState(literalPredicate.isConstant());
+ if (!literalPredicate.isConstant()) {
+ throw new NotImplementedException(
+ "Unsupported non-deterministic predicate: " + predicate_.toSql());
+ }
return literalPredicate;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
index b8eb7ed..046240f 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
@@ -42,7 +42,7 @@ import org.apache.impala.analysis.TupleDescriptor;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.rewrite.BetweenToCompoundRule;
import org.apache.impala.rewrite.ExprRewriter;
import org.slf4j.Logger;
@@ -94,7 +94,7 @@ public class HdfsPartitionPruner {
*/
public List<HdfsPartition> prunePartitions(
Analyzer analyzer, List<Expr> conjuncts, boolean allowEmpty)
- throws InternalException, AnalysisException {
+ throws ImpalaException {
// Start with creating a collection of partition filters for the applicable conjuncts.
List<HdfsPartitionFilter> partitionFilters = Lists.newArrayList();
// Conjuncts that can be evaluated from the partition key values.
@@ -438,7 +438,7 @@ public class HdfsPartitionPruner {
* filters that could not be evaluated from the partition key values.
*/
private void evalPartitionFiltersInBe(List<HdfsPartitionFilter> filters,
- HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws InternalException {
+ HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws ImpalaException {
Map<Long, HdfsPartition> partitionMap = tbl_.getPartitionMap();
// Set of partition ids that pass a filter
HashSet<Long> matchingIds = Sets.newHashSet();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
index 606425b..83f1b46 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
@@ -1019,3 +1019,14 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional.alltypes]
partitions=0/24 files=0 size=0B
====
+# IMPALA-4592: Test that we bail on evaluating non-deterministic predicates when trying
+# to prune partitions.
+select * from functional.alltypes where rand() > 100
+---- PLAN
+not implemented: Unsupported non-deterministic predicate: rand() > 100
+====
+# IMPALA-4592: Same as above but the predicate references a partition column.
+select * from functional.alltypes where rand() > year
+---- PLAN
+not implemented: Unsupported non-deterministic predicate: rand() > year
+====