You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2023/11/01 11:28:51 UTC
(flink) 02/02: [FLINK-32426][table-runtime] Fix adaptive local hash can't work when auxGrouping exist (#22856)
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch release-1.17
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a461e949cb3068b5609b0c8921647e0cbf5e5e9e
Author: Ron <ld...@163.com>
AuthorDate: Thu Jun 29 20:25:20 2023 +0800
[FLINK-32426][table-runtime] Fix adaptive local hash can't work when auxGrouping exist (#22856)
(cherry picked from commit 3f485162a372818c1402d78bf9fb25e06ca1cdf7)
---
.../planner/codegen/ProjectionCodeGenerator.scala | 4 +++
.../codegen/agg/batch/HashAggCodeGenerator.scala | 1 +
.../runtime/batch/sql/agg/HashAggITCase.scala | 29 +++++++++++++++++++++-
3 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
index 78095999e2c..e3621176163 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala
@@ -146,9 +146,13 @@ object ProjectionCodeGenerator {
outClass: Class[_ <: RowData] = classOf[BinaryRowData],
inputTerm: String = DEFAULT_INPUT1_TERM,
aggInfos: Array[AggregateInfo],
+ auxGrouping: Array[Int],
outRecordTerm: String = DEFAULT_OUT_RECORD_TERM,
outRecordWriterTerm: String = DEFAULT_OUT_RECORD_WRITER_TERM): String = {
val fieldExprs: ArrayBuffer[GeneratedExpression] = ArrayBuffer()
+ // The auxGrouping also need to consider which is aggBuffer field
+ auxGrouping.foreach(i => fieldExprs += reuseFieldExprForAggFunc(ctx, inputType, inputTerm, i))
+
aggInfos.map {
aggInfo =>
aggInfo.function match {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
index 80881bb25ad..3b9dea7f64e 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/HashAggCodeGenerator.scala
@@ -257,6 +257,7 @@ object HashAggCodeGenerator {
classOf[BinaryRowData],
inputTerm = inputTerm,
aggInfos,
+ auxGrouping,
outRecordTerm = currentValueTerm,
outRecordWriterTerm = currentValueWriterTerm
)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
index 4ff207754b4..c74de812f5f 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/HashAggITCase.scala
@@ -21,7 +21,9 @@ import org.apache.flink.api.scala._
import org.apache.flink.table.api.DataTypes
import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions}
import org.apache.flink.table.planner.codegen.agg.batch.HashAggCodeGenerator
+import org.apache.flink.table.planner.plan.stats.FlinkStatistic
import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row
+import org.apache.flink.table.planner.runtime.utils.TestData.{data1, nullablesOfData1, type1}
import org.junit.Test
import org.junit.runner.RunWith
@@ -29,12 +31,22 @@ import org.junit.runners.Parameterized
import java.math.BigDecimal
+import scala.collection.JavaConverters._
+
/** AggregateITCase using HashAgg Operator. */
@RunWith(classOf[Parameterized])
class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
extends AggregateITCaseBase("HashAggregate") {
override def prepareAggOp(): Unit = {
+ registerCollection(
+ "AuxGroupingTable",
+ data1,
+ type1,
+ "a, b, c",
+ nullablesOfData1,
+ FlinkStatistic.builder().uniqueKeys(Set(Set("a").asJava).asJava).build())
+
tEnv.getConfig.set(ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS, "SortAgg")
if (adaptiveLocalHashAggEnable) {
tEnv.getConfig
@@ -44,7 +56,7 @@ class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
Boolean.box(true))
tEnv.getConfig.set(
HashAggCodeGenerator.TABLE_EXEC_LOCAL_HASH_AGG_ADAPTIVE_SAMPLING_THRESHOLD,
- Long.box(5L))
+ Long.box(3L))
}
}
@@ -330,6 +342,21 @@ class HashAggITCase(adaptiveLocalHashAggEnable: Boolean)
)
)
}
+
+ @Test
+ def testAdaptiveHashAggWithAuxGrouping(): Unit = {
+ checkResult(
+ "SELECT a, b, COUNT(c) FROM AuxGroupingTable GROUP BY a, b",
+ Seq(
+ row(1, "a", 1),
+ row(2, "a", 1),
+ row(3, "b", 1),
+ row(4, "b", 1),
+ row(5, "c", 1),
+ row(6, "c", 1)
+ )
+ )
+ }
}
object HashAggITCase {