You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2021/01/13 01:09:04 UTC

[impala] 17/18: IMPALA-9809: Multi-aggregation query on particular dataset crashes impalad

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch 3.x
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 4796d13f71a2322abcd2f871f108e07a9b029ea1
Author: Yongzhi Chen <yc...@cloudera.com>
AuthorDate: Mon Jun 1 16:58:13 2020 -0400

    IMPALA-9809: Multi-aggregation query on particular dataset crashes impalad
    
    In streaming-aggregation-node.cc , when replicate_input_ is true
    and num_aggs > 1, it will call AddBatchStreaming several
    times(more than 1), each time, the out_batch will be used.
    If a row is not cached, the value will be saved in the out_batch,
    and out_batch's row count will be increased.
    The row_count did not set back to 0 when next while loop. Therefore
    in out_batch, it is possible that not all the tuples are non-null.
    (For example the rows added when agg_idx = 1, only tuple with 1 not
    null; the rows added when when agg_idx = 2, only tuple with 2 not
    null). But in grouping-aggregation-ir.cc, the serialize out code is
    start from very beginning of out_batch for a agg_idx, it has good
    chance to hit null tuple.
    
    Fix the issue by only serialize the tuples being added by
    current function call.
    
    Tests:
    Manual tests
    Unit tests
    
    Change-Id: I06d73171cdc40bdbb15960573030ac7fc94a7e16
    Reviewed-on: http://gerrit.cloudera.org:8080/16019
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    (cherry picked from commit 37b5599a7a25536a6e5265788cf4e139db3b0cfa)
---
 be/src/exec/grouping-aggregator-ir.cc                    |  8 +++++---
 .../tpch/queries/min-multiple-distinct-aggs.test         | 16 ++++++++++++++++
 tests/query_test/test_aggregation.py                     |  3 +++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/be/src/exec/grouping-aggregator-ir.cc b/be/src/exec/grouping-aggregator-ir.cc
index d3c7f1a..fb70e3b 100644
--- a/be/src/exec/grouping-aggregator-ir.cc
+++ b/be/src/exec/grouping-aggregator-ir.cc
@@ -157,8 +157,8 @@ Status GroupingAggregator::AddBatchStreamingImpl(int agg_idx, bool needs_seriali
     HashTableCtx* __restrict__ ht_ctx, int remaining_capacity[PARTITION_FANOUT]) {
   DCHECK(is_streaming_preagg_);
   DCHECK(!out_batch->AtCapacity());
-
-  RowBatch::Iterator out_batch_iterator(out_batch, out_batch->num_rows());
+  const int out_batch_start = out_batch->num_rows();
+  RowBatch::Iterator out_batch_iterator(out_batch, out_batch_start);
   HashTableCtx::ExprValuesCache* expr_vals_cache = ht_ctx->expr_values_cache();
   const int num_rows = in_batch->num_rows();
   const int cache_size = expr_vals_cache->capacity();
@@ -200,7 +200,9 @@ Status GroupingAggregator::AddBatchStreamingImpl(int agg_idx, bool needs_seriali
   streaming_idx_ = 0;
 ret:
   if (needs_serialize) {
-    FOREACH_ROW(out_batch, 0, out_batch_iter) {
+    // We only serialize the added rows in this call. The rows before out_batch_start may
+    // be added for other agg_idx values.
+    FOREACH_ROW(out_batch, out_batch_start, out_batch_iter) {
       AggFnEvaluator::Serialize(agg_fn_evals_, out_batch_iter.Get()->GetTuple(agg_idx));
     }
   }
diff --git a/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test b/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test
new file mode 100644
index 0000000..d3434a2
--- /dev/null
+++ b/testdata/workloads/tpch/queries/min-multiple-distinct-aggs.test
@@ -0,0 +1,16 @@
+====
+---- QUERY
+select
+l_orderkey,
+min(case when l_shipdate >'1996-03-30' then l_shipdate else null end) as flt,
+count(distinct case when l_linenumber >70 then l_partkey end) as cl,
+count(distinct l_partkey) as cnl
+from lineitem
+group by l_orderkey order by l_orderkey limit 2;
+---- TYPES
+BIGINT,STRING,BIGINT,BIGINT
+---- RESULTS
+1,'1996-04-12',0,6
+2,'1997-01-28',0,1
+====
+
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 62e9ee7..10d63ec 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -435,3 +435,6 @@ class TestTPCHAggregationQueries(ImpalaTestSuite):
 
   def test_tpch_stress(self, vector):
     self.run_test_case('tpch-stress-aggregations', vector)
+
+  def test_min_multiple_distinct(self, vector, unique_database):
+    self.run_test_case('min-multiple-distinct-aggs', vector)