You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2023/11/01 11:28:51 UTC

(flink) 02/02: [FLINK-32426][table-runtime] Fix adaptive local hash can't work when auxGrouping exist (#22856)

This is an automated email from the ASF dual-hosted git repository.

ron pushed a commit to branch release-1.17
in repository https://gitbox.apache.org/repos/asf/flink.git

commit a461e949cb3068b5609b0c8921647e0cbf5e5e9e
Author: Ron <ld...@163.com>
AuthorDate: Thu Jun 29 20:25:20 2023 +0800

    [FLINK-32426][table-runtime] Fix adaptive local hash can't work when auxGrouping exist (#22856)
    
    (cherry picked from commit 3f485162a372818c1402d78bf9fb25e06ca1cdf7)
---
 .../planner/codegen/ProjectionCodeGenerator.scala  |  4 +++
 .../codegen/agg/batch/HashAggCodeGenerator.scala   |  1 +
 .../runtime/batch/sql/agg/HashAggITCase.scala      | 29 +++++++++++++++++++++-
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
index 78095999e2c..e3621176163 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
@@ -146,9 +146,13 @@ object ProjectionCodeGenerator {
       outClass: Class[_ <: RowData] = classOf[BinaryRowData],
       inputTerm: String = DEFAULT_INPUT1_TERM,
       aggInfos: Array[AggregateInfo],
+      auxGrouping: Array[Int],
       outRecordTerm: String = DEFAULT_OUT_RECORD_TERM,
       outRecordWriterTerm: String = DEFAULT_OUT_RECORD_WRITER_TERM): String = {
     val fieldExprs: ArrayBuffer[GeneratedExpression] = ArrayBuffer()
+    // The auxGrouping also need to consider which is aggBuffer field
+    auxGrouping.foreach(i => fieldExprs += reuseFieldExprForAggFunc(ctx, inputType, inputTerm, i))
+
     aggInfos.map {
       aggInfo =>
         aggInfo.function match {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
index 80881bb25ad..3b9dea7f64e 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
@@ -257,6 +257,7 @@ object HashAggCodeGenerator {
           classOf[BinaryRowData],
           inputTerm = inputTerm,
           aggInfos,
+          auxGrouping,
           outRecordTerm = currentValueTerm,
           outRecordWriterTerm = currentValueWriterTerm
         )
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
index 4ff207754b4..c74de812f5f 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
@@ -21,7 +21,9 @@ import org.apache.flink.api.scala._
 import org.apache.flink.table.api.DataTypes
 import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions}
 import org.apache.flink.table.planner.codegen.agg.batch.HashAggCodeGenerator
+import org.apache.flink.table.planner.plan.stats.FlinkStatistic
 import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row
+import org.apache.flink.table.planner.runtime.utils.TestData.{data1, nullablesOfData1, type1}
 
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -29,12 +31,22 @@ import org.junit.runners.Parameterized
 
 import java.math.BigDecimal
 
+import scala.collection.JavaConverters._
+
 /** AggregateITCase using HashAgg Operator. */
 @RunWith(classOf[Parameterized])
 class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
   extends AggregateITCaseBase("HashAggregate") {
 
   override def prepareAggOp(): Unit = {
+    registerCollection(
+      "AuxGroupingTable",
+      data1,
+      type1,
+      "a, b, c",
+      nullablesOfData1,
+      FlinkStatistic.builder().uniqueKeys(Set(Set("a").asJava).asJava).build())
+
     tEnv.getConfig.set(ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS, "SortAgg")
     if (adaptiveLocalHashAggEnable) {
       tEnv.getConfig
@@ -44,7 +56,7 @@ class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
         Boolean.box(true))
       tEnv.getConfig.set(
         HashAggCodeGenerator.TABLE_EXEC_LOCAL_HASH_AGG_ADAPTIVE_SAMPLING_THRESHOLD,
-        Long.box(5L))
+        Long.box(3L))
     }
   }
 
@@ -330,6 +342,21 @@ class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
       )
     )
   }
+
+  @Test
+  def testAdaptiveHashAggWithAuxGrouping(): Unit = {
+    checkResult(
+      "SELECT a, b, COUNT(c) FROM AuxGroupingTable GROUP BY a, b",
+      Seq(
+        row(1, "a", 1),
+        row(2, "a", 1),
+        row(3, "b", 1),
+        row(4, "b", 1),
+        row(5, "c", 1),
+        row(6, "c", 1)
+      )
+    )
+  }
 }
 
 object HashAggITCase {