You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2018/10/24 17:37:12 UTC
[2/2] impala git commit: IMPALA-7677: Fix DCHECK failure in
GroupingAggregator
IMPALA-7677: Fix DCHECK failure in GroupingAggregator
After inserting all of its input into its Aggregators,
StreamingAggregationNode performs some cleanup, such as calling
InputDone() on each Aggregator.
Previously, StreamingAggregationNode only checked that all of the
child's batches had been fetched before doing this cleanup, which
causes problems if the final child batch isn't processed fully in a
single GetNext() call. In this case, multiple calls to InputDone()
lead to a DCHECK failure.
The solution is to only perform the cleanup once the final child batch
has been fully processed.
Testing:
- Added an e2e test with a query that hits this condition.
Change-Id: I851007a60472d0e53081c076c863c866c516677c
Reviewed-on: http://gerrit.cloudera.org:8080/11626
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/15e8ce4f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/15e8ce4f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/15e8ce4f
Branch: refs/heads/master
Commit: 15e8ce4f273945ce548fe677ee0140dea8068e6d
Parents: e00c082
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Tue Oct 9 17:08:13 2018 +0000
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed Oct 24 01:03:38 2018 +0000
----------------------------------------------------------------------
be/src/exec/streaming-aggregation-node.cc | 2 +-
.../queries/QueryTest/multiple-distinct-aggs.test | 15 +++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/15e8ce4f/be/src/exec/streaming-aggregation-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/streaming-aggregation-node.cc b/be/src/exec/streaming-aggregation-node.cc
index 7cbebb8..42f3e9f 100644
--- a/be/src/exec/streaming-aggregation-node.cc
+++ b/be/src/exec/streaming-aggregation-node.cc
@@ -171,7 +171,7 @@ Status StreamingAggregationNode::GetRowsStreaming(
child_batch_->Reset();
} while (out_batch->num_rows() == 0 && !child_eos_);
- if (child_eos_) {
+ if (child_eos_ && child_batch_processed_) {
child(0)->Close(state);
child_batch_.reset();
for (auto& agg : aggs_) RETURN_IF_ERROR(agg->InputDone());
http://git-wip-us.apache.org/repos/asf/impala/blob/15e8ce4f/testdata/workloads/functional-query/queries/QueryTest/multiple-distinct-aggs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/multiple-distinct-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/multiple-distinct-aggs.test
index b3ac83d..5ae87d8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/multiple-distinct-aggs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/multiple-distinct-aggs.test
@@ -422,3 +422,18 @@ BIGINT,BIGINT,BIGINT,BIGINT,BIGINT,BIGINT,BIGINT,BIGINT,BIGINT
---- RESULTS
8,2,2,2,2,2,2,2,8
====
+---- QUERY
+# IMPALA-7677: query that hits the path in StreamingAggregationNode where the final child
+# batch is not fully processed in a single GetNext() call.
+select
+ count(distinct o_totalprice),
+ count(distinct o_orderkey),
+ count(distinct o_orderdate)
+from tpch.orders
+where 1 not in
+ (select count(distinct o_custkey) from tpch.orders)
+---- TYPES
+BIGINT,BIGINT,BIGINT
+---- RESULTS
+1464556,1500000,2406
+====
\ No newline at end of file