You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/09/11 20:41:21 UTC
[impala] 02/02: IMPALA-8718: project out collection slots in
analytic's sort tuple
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit bb5d3c255e12478e30405bccaa4fd202393471c1
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Sep 9 15:14:47 2019 +0800
IMPALA-8718: project out collection slots in analytic's sort tuple
Subplan node is mainly used to extract collection values. It evaluates
its right plan tree (usually a nested loop join) for every row from its
left child (usually a scan producing tuples with collection values), and
returns those rows produced by the right child. Each row (TupleRow)
produced by the join node consists of several tuples from the join
operands. So the scan node tuple that contains collection values will be
part of the output of the join node, then become part of the output of
the subplan node.
When generating analytic plan, a TupleDescriptor for sort is created
based on the materialized slots of the input. If the input comes from a
subplan node, there are collection slots in it. These collection slots
will be picked out into the sort tuple, and occur in the smap of it.
Then the output smap of the analytic plan will contain the collection
slot consequently. This causes IllegalStateException if the analytic
plan is the nullable side of an outer join. The exception is thrown when
we are checking the necessary of adding a TupleIsNullPredicate for each
output slot.
We should project out the collection slots in creating the sort tuple of
analytic plan to avoid causing such an exception. Projecting out them is
safe since outputs of the analytic node must be in the select list of
the block with the analytic, and we don't allow collection types to be
returned from a select block, and also don't support any builtin or UDF
functions that take collection types as an argument.
Tests
- Add Planner test in analytic-fns.test with VALIDATE_CARDINALITY
enabled. Also fix some incorrect row-sizes of existing tests.
- Add e2e test in nested-types-runtime.test to verify that collection
slots are projected out.
Change-Id: I7edf74ff0f603dfd33ff546e61545bc724990655
Reviewed-on: http://gerrit.cloudera.org:8080/14135
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/analysis/TupleIsNullPredicate.java | 2 +
.../org/apache/impala/planner/AnalyticPlanner.java | 13 +++
.../org/apache/impala/planner/PlannerTest.java | 3 +-
.../queries/PlannerTest/analytic-fns.test | 105 ++++++++++++++++++---
.../queries/QueryTest/nested-types-runtime.test | 28 ++++++
5 files changed, 137 insertions(+), 14 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java b/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
index 0605ff9..1562a15 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
@@ -156,6 +156,8 @@ public class TupleIsNullPredicate extends Predicate {
private static boolean requiresNullWrapping(Expr expr, Analyzer analyzer)
throws InternalException {
Preconditions.checkNotNull(expr);
+ Preconditions.checkState(!expr.getType().isComplexType(),
+ "Should not evaluate on complex type: " + expr.debugString());
// If the expr is already wrapped in an IF(TupleIsNull(), NULL, expr)
// then it must definitely be wrapped again at this level.
// Do not try to execute expr because a TupleIsNullPredicate is not constant.
diff --git a/fe/src/main/java/org/apache/impala/planner/AnalyticPlanner.java b/fe/src/main/java/org/apache/impala/planner/AnalyticPlanner.java
index af2a871..598d483 100644
--- a/fe/src/main/java/org/apache/impala/planner/AnalyticPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/AnalyticPlanner.java
@@ -284,6 +284,19 @@ public class AnalyticPlanner {
TupleDescriptor tupleDesc = analyzer_.getTupleDesc(tid);
for (SlotDescriptor inputSlotDesc: tupleDesc.getSlots()) {
if (!inputSlotDesc.isMaterialized()) continue;
+ if (inputSlotDesc.getType().isComplexType()) {
+ // Project out collection slots since they won't be used anymore and may cause
+ // troubles like IMPALA-8718. They won't be used since outputs of the analytic
+ // node must be in the select list of the block with the analytic, and we don't
+ // allow collection types to be returned from a select block, and also don't
+ // support any builtin or UDF functions that take collection types as an
+ // argument.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Project out collection slot in sort tuple of analytic: slot={}",
+ inputSlotDesc.debugString());
+ }
+ continue;
+ }
inputSlotRefs.add(new SlotRef(inputSlotDesc));
}
}
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index cf6e802..481973e 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -167,7 +167,8 @@ public class PlannerTest extends PlannerTestBase {
@Test
public void testAnalyticFns() {
- runPlannerTestFile("analytic-fns");
+ runPlannerTestFile("analytic-fns",
+ ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
}
@Test
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test b/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
index b4856f3..ee1bead 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
@@ -1329,11 +1329,11 @@ PLAN-ROOT SINK
|
12:SORT
| order by: min(id) OVER(...) DESC NULLS FIRST
-| row-size=4B cardinality=7.41K
+| row-size=8B cardinality=7.41K
|
08:UNION
| pass-through-operands: 07
-| row-size=4B cardinality=7.41K
+| row-size=8B cardinality=7.41K
|
|--11:ANALYTIC
| | functions: sum(bigint_col)
@@ -1352,10 +1352,10 @@ PLAN-ROOT SINK
|
07:AGGREGATE [FINALIZE]
| group by: min(id) OVER(...)
-| row-size=4B cardinality=7.40K
+| row-size=8B cardinality=7.40K
|
00:UNION
-| row-size=4B cardinality=7.40K
+| row-size=8B cardinality=7.40K
|
|--06:ANALYTIC
| | functions: max(id)
@@ -1390,11 +1390,11 @@ PLAN-ROOT SINK
|
12:SORT
| order by: min(id) OVER(...) DESC NULLS FIRST
-| row-size=4B cardinality=7.41K
+| row-size=8B cardinality=7.41K
|
08:UNION
| pass-through-operands: 16
-| row-size=4B cardinality=7.41K
+| row-size=8B cardinality=7.41K
|
|--11:ANALYTIC
| | functions: sum(bigint_col)
@@ -1415,16 +1415,16 @@ PLAN-ROOT SINK
|
16:AGGREGATE [FINALIZE]
| group by: min(id) OVER(...)
-| row-size=4B cardinality=7.40K
+| row-size=8B cardinality=7.40K
|
15:EXCHANGE [HASH(min(id) OVER(...))]
|
07:AGGREGATE [STREAMING]
| group by: min(id) OVER(...)
-| row-size=4B cardinality=7.40K
+| row-size=8B cardinality=7.40K
|
00:UNION
-| row-size=4B cardinality=7.40K
+| row-size=8B cardinality=7.40K
|
|--06:ANALYTIC
| | functions: max(id)
@@ -2086,10 +2086,10 @@ PLAN-ROOT SINK
|
05:SELECT
| predicates: id IS NULL, tinyint_col != 5
-| row-size=5B cardinality=730
+| row-size=9B cardinality=730
|
00:UNION
-| row-size=5B cardinality=7.30K
+| row-size=9B cardinality=7.30K
|
|--04:ANALYTIC
| | functions: dense_rank()
@@ -2124,10 +2124,10 @@ PLAN-ROOT SINK
|
05:SELECT
| predicates: id IS NULL, tinyint_col != 5
-| row-size=5B cardinality=730
+| row-size=9B cardinality=730
|
00:UNION
-| row-size=5B cardinality=7.30K
+| row-size=9B cardinality=7.30K
|
|--08:EXCHANGE [RANDOM]
| |
@@ -3011,3 +3011,82 @@ PLAN-ROOT SINK
HDFS partitions=24/24 files=24 size=478.45KB
row-size=25B cardinality=7.30K
====
+# IMPALA-8718: Collection slots are removed by the projection in creating sort tuple of
+# analytic node. So row size of sort shrinks to 8B. Output row size of analytic shrinks
+# to 16B.
+select t.c_custkey, rank() over(order by t.c_custkey) as rnk
+from tpch_nested_parquet.customer t left outer join t.c_orders
+---- PLAN
+PLAN-ROOT SINK
+|
+06:ANALYTIC
+| functions: rank()
+| order by: c_custkey ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=16B cardinality=150.00K
+|
+05:SORT
+| order by: c_custkey ASC
+| row-size=8B cardinality=150.00K
+|
+01:SUBPLAN
+| row-size=20B cardinality=150.00K
+|
+|--04:NESTED LOOP JOIN [RIGHT OUTER JOIN]
+| | row-size=20B cardinality=1
+| |
+| |--02:SINGULAR ROW SRC
+| | row-size=20B cardinality=1
+| |
+| 03:UNNEST [t.c_orders]
+| row-size=0B cardinality=10
+|
+00:SCAN HDFS [tpch_nested_parquet.customer t]
+ HDFS partitions=1/1 files=4 size=292.39MB
+ row-size=20B cardinality=150.00K
+====
+# IMPALA-8718: No more collection slots in the output of the inline view of analytics.
+select leftSide.c_custkey, c_name, rnk
+from tpch_nested_parquet.customer leftSide
+left outer join (
+ select t.c_custkey, rank() over(order by t.c_custkey) as rnk
+ from tpch_nested_parquet.customer t left outer join t.c_orders
+) rightSide
+on leftSide.c_custkey = rightSide.c_custkey
+---- PLAN
+PLAN-ROOT SINK
+|
+08:HASH JOIN [LEFT OUTER JOIN]
+| hash predicates: leftSide.c_custkey = c_custkey
+| row-size=54B cardinality=150.00K
+|
+|--07:ANALYTIC
+| | functions: rank()
+| | order by: c_custkey ASC
+| | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| | row-size=16B cardinality=150.00K
+| |
+| 06:SORT
+| | order by: c_custkey ASC
+| | row-size=8B cardinality=150.00K
+| |
+| 02:SUBPLAN
+| | row-size=20B cardinality=150.00K
+| |
+| |--05:NESTED LOOP JOIN [RIGHT OUTER JOIN]
+| | | row-size=20B cardinality=1
+| | |
+| | |--03:SINGULAR ROW SRC
+| | | row-size=20B cardinality=1
+| | |
+| | 04:UNNEST [t.c_orders]
+| | row-size=0B cardinality=10
+| |
+| 01:SCAN HDFS [tpch_nested_parquet.customer t]
+| HDFS partitions=1/1 files=4 size=289.00MB
+| row-size=20B cardinality=150.00K
+|
+00:SCAN HDFS [tpch_nested_parquet.customer leftside]
+ HDFS partitions=1/1 files=4 size=289.00MB
+ row-size=38B cardinality=150.00K
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-runtime.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-runtime.test
index c72558a..1f3b966 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-runtime.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-runtime.test
@@ -466,3 +466,31 @@ select id, m from complextypestbl t,
---- TYPES
BIGINT,STRING
====
+---- QUERY
+# IMPALA-8718: Test collection slots are not passed outside the analytic node.
+select leftSide.id, rnk
+from complextypestbl leftSide
+left outer join (
+ select t.id, rank() over (order by t.id) as rnk
+ from complextypestbl t left outer join t.int_array
+) rightSide
+on leftSide.id = rightSide.id;
+---- RESULTS
+1,1
+1,1
+1,1
+2,4
+2,4
+2,4
+2,4
+2,4
+2,4
+3,10
+4,11
+5,12
+6,13
+7,14
+8,15
+---- TYPES
+BIGINT,BIGINT
+====