You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2021/01/13 01:09:04 UTC
[impala] 17/18: IMPALA-9809: Multi-aggregation query on particular
dataset crashes impalad
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch 3.x
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 4796d13f71a2322abcd2f871f108e07a9b029ea1
Author: Yongzhi Chen <yc...@cloudera.com>
AuthorDate: Mon Jun 1 16:58:13 2020 -0400
IMPALA-9809: Multi-aggregation query on particular dataset crashes impalad
In streaming-aggregation-node.cc , when replicate_input_ is true
and num_aggs > 1, it will call AddBatchStreaming several
times(more than 1), each time, the out_batch will be used.
If a row is not cached, the value will be saved in the out_batch,
and out_batch's row count will be increased.
The row_count did not set back to 0 when next while loop. Therefore
in out_batch, it is possible that not all the tuples are non-null.
(For example the rows added when agg_idx = 1, only tuple with 1 not
null; the rows added when when agg_idx = 2, only tuple with 2 not
null). But in grouping-aggregation-ir.cc, the serialize out code is
start from very beginning of out_batch for a agg_idx, it has good
chance to hit null tuple.
Fix the issue by only serialize the tuples being added by
current function call.
Tests:
Manual tests
Unit tests
Change-Id: I06d73171cdc40bdbb15960573030ac7fc94a7e16
Reviewed-on: http://gerrit.cloudera.org:8080/16019
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
(cherry picked from commit 37b5599a7a25536a6e5265788cf4e139db3b0cfa)
---
be/src/exec/grouping-aggregator-ir.cc | 8 +++++---
.../tpch/queries/min-multiple-distinct-aggs.test | 16 ++++++++++++++++
tests/query_test/test_aggregation.py | 3 +++
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/be/src/exec/grouping-aggregator-ir.cc b/be/src/exec/grouping-aggregator-ir.cc
index d3c7f1a..fb70e3b 100644
--- a/be/src/exec/grouping-aggregator-ir.cc
+++ b/be/src/exec/grouping-aggregator-ir.cc
@@ -157,8 +157,8 @@ Status GroupingAggregator::AddBatchStreamingImpl(int agg_idx, bool needs_seriali
HashTableCtx* __restrict__ ht_ctx, int remaining_capacity[PARTITION_FANOUT]) {
DCHECK(is_streaming_preagg_);
DCHECK(!out_batch->AtCapacity());
-
- RowBatch::Iterator out_batch_iterator(out_batch, out_batch->num_rows());
+ const int out_batch_start = out_batch->num_rows();
+ RowBatch::Iterator out_batch_iterator(out_batch, out_batch_start);
HashTableCtx::ExprValuesCache* expr_vals_cache = ht_ctx->expr_values_cache();
const int num_rows = in_batch->num_rows();
const int cache_size = expr_vals_cache->capacity();
@@ -200,7 +200,9 @@ Status GroupingAggregator::AddBatchStreamingImpl(int agg_idx, bool needs_seriali
streaming_idx_ = 0;
ret:
if (needs_serialize) {
- FOREACH_ROW(out_batch, 0, out_batch_iter) {
+ // We only serialize the added rows in this call. The rows before out_batch_start may
+ // be added for other agg_idx values.
+ FOREACH_ROW(out_batch, out_batch_start, out_batch_iter) {
AggFnEvaluator::Serialize(agg_fn_evals_, out_batch_iter.Get()->GetTuple(agg_idx));
}
}
diff --git a/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test b/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test
new file mode 100644
index 0000000..d3434a2
--- /dev/null
+++ b/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test
@@ -0,0 +1,16 @@
+====
+---- QUERY
+select
+l_orderkey,
+min(case when l_shipdate >'1996-03-30' then l_shipdate else null end) as flt,
+count(distinct case when l_linenumber >70 then l_partkey end) as cl,
+count(distinct l_partkey) as cnl
+from lineitem
+group by l_orderkey order by l_orderkey limit 2;
+---- TYPES
+BIGINT,STRING,BIGINT,BIGINT
+---- RESULTS
+1,'1996-04-12',0,6
+2,'1997-01-28',0,1
+====
+
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 62e9ee7..10d63ec 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -435,3 +435,6 @@ class TestTPCHAggregationQueries(ImpalaTestSuite):
def test_tpch_stress(self, vector):
self.run_test_case('tpch-stress-aggregations', vector)
+
+ def test_min_multiple_distinct(self, vector, unique_database):
+ self.run_test_case('min-multiple-distinct-aggs', vector)