You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2025/01/14 11:01:44 UTC
(flink) 03/04: [FLINK-36629][table-planner] Modify AdaptiveJoinProcessor to support adaptive skewed join optimization
This is an automated email from the ASF dual-hosted git repository.
zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit ab1649e9500c05f1d1478f71c3e0d291d1573de3
Author: noorall <86...@qq.com>
AuthorDate: Fri Jan 10 23:06:51 2025 +0800
[FLINK-36629][table-planner] Modify AdaptiveJoinProcessor to support adaptive skewed join optimization
---
.../exec/processor/AdaptiveJoinProcessor.java | 5 +
.../table/planner/delegation/BatchPlanner.scala | 28 ++-
.../utils/AdaptiveBatchAbstractTestBase.java | 51 ++++++
.../plan/batch/sql/adaptive/AdaptiveJoinTest.xml | 23 +++
.../plan/batch/sql/adaptive/AdaptiveJoinTest.scala | 12 ++
.../sql/adaptive/AdaptiveBroadcastJoinITCase.scala | 153 +++-------------
...stJoinITCase.scala => AdaptiveJoinITCase.scala} | 74 ++------
.../sql/adaptive/AdaptiveSkewedJoinITCase.scala | 102 +++++++++++
.../runtime/batch/sql/join/JoinITCase.scala | 3 +
.../runtime/utils/AdaptiveBatchTestBase.scala | 98 ++++++++++
.../planner/runtime/utils/BatchTableEnvUtil.scala | 9 +-
.../planner/runtime/utils/BatchTestBase.scala | 198 ++++++++++++---------
.../flink/table/planner/utils/TableTestBase.scala | 3 +
.../AdaptiveBroadcastJoinOptimizationStrategy.java | 20 ++-
.../strategy/AdaptiveJoinOptimizationUtils.java | 62 +++++++
15 files changed, 555 insertions(+), 286 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/AdaptiveJoinProcessor.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/AdaptiveJoinProcessor.java
index c71540fc17e..7187eabf589 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/AdaptiveJoinProcessor.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/AdaptiveJoinProcessor.java
@@ -142,6 +142,11 @@ public class AdaptiveJoinProcessor implements ExecNodeGraphProcessor {
!= OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE
&& !TableConfigUtils.isOperatorDisabled(
tableConfig, OperatorType.BroadcastHashJoin);
+ isAdaptiveJoinEnabled |=
+ tableConfig.get(
+ OptimizerConfigOptions
+ .TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY)
+ != OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE;
JobManagerOptions.SchedulerType schedulerType =
context.getPlanner()
.getExecEnv()
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/delegation/BatchPlanner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/delegation/BatchPlanner.scala
index 6686de333ee..132d5ab7176 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/delegation/BatchPlanner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/delegation/BatchPlanner.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.planner.delegation
import org.apache.flink.api.common.RuntimeExecutionMode
import org.apache.flink.api.dag.Transformation
import org.apache.flink.configuration.ExecutionOptions
-import org.apache.flink.runtime.scheduler.adaptivebatch.StreamGraphOptimizationStrategy
+import org.apache.flink.runtime.scheduler.adaptivebatch.{BatchExecutionOptionsInternal, StreamGraphOptimizationStrategy}
import org.apache.flink.table.api._
import org.apache.flink.table.api.config.OptimizerConfigOptions
import org.apache.flink.table.catalog.{CatalogManager, FunctionCatalog}
@@ -35,7 +35,7 @@ import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodePlanDumper
import org.apache.flink.table.planner.plan.optimize.{BatchCommonSubGraphBasedOptimizer, Optimizer}
import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil
import org.apache.flink.table.planner.utils.DummyStreamExecutionEnvironment
-import org.apache.flink.table.runtime.strategy.{AdaptiveBroadcastJoinOptimizationStrategy, PostProcessAdaptiveJoinStrategy}
+import org.apache.flink.table.runtime.strategy.{AdaptiveBroadcastJoinOptimizationStrategy, AdaptiveSkewedJoinOptimizationStrategy, PostProcessAdaptiveJoinStrategy}
import org.apache.calcite.plan.{ConventionTraitDef, RelTrait, RelTraitDef}
import org.apache.calcite.rel.RelCollationTraitDef
@@ -106,11 +106,27 @@ class BatchPlanner(
super.afterTranslation()
val configuration = getTableConfig
val optimizationStrategies = new util.ArrayList[String]()
- if (
- configuration.get(OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY)
- != OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE
- ) {
+ val isAdaptiveBroadcastJoinEnabled = configuration.get(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY) != OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE
+ val isAdaptiveSkewedJoinEnabled = configuration.get(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY) != OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE
+ if (isAdaptiveBroadcastJoinEnabled) {
optimizationStrategies.add(classOf[AdaptiveBroadcastJoinOptimizationStrategy].getName)
+ }
+ if (isAdaptiveSkewedJoinEnabled) {
+ optimizationStrategies.add(classOf[AdaptiveSkewedJoinOptimizationStrategy].getName)
+ configuration.set(
+ BatchExecutionOptionsInternal.ADAPTIVE_SKEWED_OPTIMIZATION_SKEWED_FACTOR,
+ configuration.get(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_SKEWED_FACTOR)
+ )
+ configuration.set(
+ BatchExecutionOptionsInternal.ADAPTIVE_SKEWED_OPTIMIZATION_SKEWED_THRESHOLD,
+ configuration.get(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_SKEWED_THRESHOLD)
+ )
+ }
+ if (isAdaptiveBroadcastJoinEnabled || isAdaptiveSkewedJoinEnabled) {
optimizationStrategies.add(classOf[PostProcessAdaptiveJoinStrategy].getName)
}
configuration.set(
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchAbstractTestBase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchAbstractTestBase.java
new file mode 100644
index 00000000000..fd0e7730c17
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchAbstractTestBase.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.runtime.utils;
+
+import org.apache.flink.configuration.BatchExecutionOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Adaptive batch test base to use {@link RegisterExtension}. */
+public class AdaptiveBatchAbstractTestBase {
+ protected static final int DEFAULT_PARALLELISM = 3;
+
+ @RegisterExtension
+ private static final MiniClusterExtension MINI_CLUSTER_EXTENSION =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setConfiguration(getConfiguration())
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(DEFAULT_PARALLELISM)
+ .build());
+
+ private static Configuration getConfiguration() {
+ Configuration config = new Configuration();
+ config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("100m"));
+ config.set(
+ BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_AVG_DATA_VOLUME_PER_TASK,
+ MemorySize.parse("100k"));
+ return config;
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.xml
index fd21759b546..4931121cbc8 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.xml
@@ -246,6 +246,29 @@ LogicalProject(a1=[$0], b1=[$1], c1=[$2], d1=[$3], a2=[$4], b2=[$5], c2=[$6], d2
HashJoin(joinType=[InnerJoin], where=[(a1 = a2)], select=[a1, b1, c1, d1, a2, b2, c2, d2], build=[left])
:- Exchange(distribution=[hash[a1]])
: +- TableSourceScan(table=[[default_catalog, default_database, T1]], fields=[a1, b1, c1, d1])
++- Exchange(distribution=[hash[a2]])
+ +- TableSourceScan(table=[[default_catalog, default_database, T2]], fields=[a2, b2, c2, d2])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testAdaptiveJoinWithAdaptiveSkewedOptimizationEnabled">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM T1, T2 WHERE a1 = a2]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(a1=[$0], b1=[$1], c1=[$2], d1=[$3], a2=[$4], b2=[$5], c2=[$6], d2=[$7])
++- LogicalFilter(condition=[=($0, $4)])
+ +- LogicalJoin(condition=[true], joinType=[inner])
+ :- LogicalTableScan(table=[[default_catalog, default_database, T1]])
+ +- LogicalTableScan(table=[[default_catalog, default_database, T2]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+AdaptiveJoin(originalJoin=[ShuffleHashJoin], joinType=[InnerJoin], where=[(a1 = a2)], select=[a1, b1, c1, d1, a2, b2, c2, d2], build=[left])
+:- Exchange(distribution=[hash[a1]])
+: +- TableSourceScan(table=[[default_catalog, default_database, T1]], fields=[a1, b1, c1, d1])
+- Exchange(distribution=[hash[a2]])
+- TableSourceScan(table=[[default_catalog, default_database, T2]], fields=[a2, b2, c2, d2])
]]>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.scala
index 2d00197a3b8..4e380ef8dc2 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/adaptive/AdaptiveJoinTest.scala
@@ -133,4 +133,16 @@ class AdaptiveJoinTest extends TableTestBase {
val sql = "SELECT * FROM T1, T2 WHERE a1 = a2"
util.verifyExecPlan(sql)
}
+
+ @Test
+ def testAdaptiveJoinWithAdaptiveSkewedOptimizationEnabled(): Unit = {
+ util.tableConfig.set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
+ OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE)
+ util.tableConfig.set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.AUTO)
+ val sql = "SELECT * FROM T1, T2 WHERE a1 = a2"
+ util.verifyExecPlan(sql)
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala
index 47c95fbc3ff..74ef3162335 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala
@@ -17,136 +17,48 @@
*/
package org.apache.flink.table.planner.runtime.batch.sql.adaptive
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{LONG_TYPE_INFO, STRING_TYPE_INFO}
-import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions}
-import org.apache.flink.table.planner.runtime.utils.BatchTestBase
-import org.apache.flink.types.Row
+import org.apache.flink.table.api.config.OptimizerConfigOptions
-import org.junit.jupiter.api.{BeforeEach, Test}
-
-import scala.collection.JavaConversions._
-import scala.util.Random
+import org.junit.jupiter.api.BeforeEach
/** IT cases for adaptive broadcast join. */
-class AdaptiveBroadcastJoinITCase extends BatchTestBase {
-
+class AdaptiveBroadcastJoinITCase extends AdaptiveJoinITCase {
@BeforeEach
override def before(): Unit = {
super.before()
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE)
+
registerCollection(
"T",
- AdaptiveBroadcastJoinITCase.dataT,
- AdaptiveBroadcastJoinITCase.rowType,
+ AdaptiveJoinITCase.generateRandomData,
+ AdaptiveJoinITCase.rowType,
"a, b, c, d",
- AdaptiveBroadcastJoinITCase.nullables)
+ AdaptiveJoinITCase.nullables)
registerCollection(
"T1",
- AdaptiveBroadcastJoinITCase.dataT1,
- AdaptiveBroadcastJoinITCase.rowType,
+ AdaptiveJoinITCase.generateRandomData,
+ AdaptiveJoinITCase.rowType,
"a1, b1, c1, d1",
- AdaptiveBroadcastJoinITCase.nullables)
+ AdaptiveJoinITCase.nullables)
registerCollection(
"T2",
- AdaptiveBroadcastJoinITCase.dataT2,
- AdaptiveBroadcastJoinITCase.rowType,
+ AdaptiveJoinITCase.generateRandomData,
+ AdaptiveJoinITCase.rowType,
"a2, b2, c2, d2",
- AdaptiveBroadcastJoinITCase.nullables)
+ AdaptiveJoinITCase.nullables)
registerCollection(
"T3",
- AdaptiveBroadcastJoinITCase.dataT3,
- AdaptiveBroadcastJoinITCase.rowType,
+ AdaptiveJoinITCase.generateRandomData,
+ AdaptiveJoinITCase.rowType,
"a3, b3, c3, d3",
- AdaptiveBroadcastJoinITCase.nullables)
- }
-
- @Test
- def testWithShuffleHashJoin(): Unit = {
- tEnv.getConfig
- .set(ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS, "NestedLoopJoin,SortMergeJoin")
- testSimpleJoin()
+ AdaptiveJoinITCase.nullables)
}
- @Test
- def testWithShuffleMergeJoin(): Unit = {
- tEnv.getConfig
- .set(ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS, "NestedLoopJoin,ShuffleHashJoin")
- testSimpleJoin()
- }
-
- @Test
- def testWithBroadcastJoin(): Unit = {
- tEnv.getConfig.set(
- ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS,
- "SortMergeJoin,NestedLoopJoin")
- tEnv.getConfig.set(
- OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD,
- Long.box(Long.MaxValue))
- testSimpleJoin()
- }
-
- @Test
- def testShuffleJoinWithForwardForConsecutiveHash(): Unit = {
- tEnv.getConfig.set(
- OptimizerConfigOptions.TABLE_OPTIMIZER_MULTIPLE_INPUT_ENABLED,
- Boolean.box(false))
- val sql =
- """
- |WITH
- | r AS (SELECT * FROM T1, T2, T3 WHERE a1 = a2 and a1 = a3)
- |SELECT sum(b1) FROM r group by a1
- |""".stripMargin
- checkResult(sql)
- }
-
- @Test
- def testJoinWithUnionInput(): Unit = {
- val sql =
- """
- |SELECT * FROM
- | (SELECT a FROM (SELECT a1 as a FROM T1) UNION ALL (SELECT a2 as a FROM T2)) Y
- | LEFT JOIN T ON T.a = Y.a
- |""".stripMargin
- checkResult(sql)
- }
-
- @Test
- def testJoinWithMultipleInput(): Unit = {
- val sql =
- """
- |SELECT * FROM
- | (SELECT a FROM T1 JOIN T ON a = a1) t1
- | INNER JOIN
- | (SELECT d2 FROM T JOIN T2 ON d2 = a) t2
- |ON t1.a = t2.d2
- |""".stripMargin
- checkResult(sql)
- }
-
- def testSimpleJoin(): Unit = {
- // inner join
- val sql1 = "SELECT * FROM T1, T2 WHERE a1 = a2"
- checkResult(sql1)
-
- // left join
- val sql2 = "SELECT * FROM T1 LEFT JOIN T2 on a1 = a2"
- checkResult(sql2)
-
- // right join
- val sql3 = "SELECT * FROM T1 RIGHT JOIN T2 on a1 = a2"
- checkResult(sql3)
-
- // semi join
- val sql4 = "SELECT * FROM T1 WHERE a1 IN (SELECT a2 FROM T2)"
- checkResult(sql4)
-
- // anti join
- val sql5 = "SELECT * FROM T1 WHERE a1 NOT IN (SELECT a2 FROM T2 where a2 = a1)"
- checkResult(sql5)
- }
-
- def checkResult(sql: String): Unit = {
+ override def checkResult(sql: String): Unit = {
tEnv.getConfig
.set(
OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
@@ -164,26 +76,3 @@ class AdaptiveBroadcastJoinITCase extends BatchTestBase {
checkResult(sql, expected)
}
}
-
-object AdaptiveBroadcastJoinITCase {
-
- def generateRandomData(): Seq[Row] = {
- val data = new java.util.ArrayList[Row]()
- val numRows = Random.nextInt(30)
- lazy val strs = Seq("adaptive", "join", "itcase")
- for (x <- 0 until numRows) {
- data.add(
- BatchTestBase.row(x.toLong, Random.nextLong(), strs(Random.nextInt(3)), Random.nextLong()))
- }
- data
- }
-
- lazy val rowType =
- new RowTypeInfo(LONG_TYPE_INFO, LONG_TYPE_INFO, STRING_TYPE_INFO, LONG_TYPE_INFO)
- lazy val nullables: Array[Boolean] = Array(true, true, true, true)
-
- lazy val dataT: Seq[Row] = generateRandomData()
- lazy val dataT1: Seq[Row] = generateRandomData()
- lazy val dataT2: Seq[Row] = generateRandomData()
- lazy val dataT3: Seq[Row] = generateRandomData()
-}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveJoinITCase.scala
similarity index 65%
copy from flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala
copy to flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveJoinITCase.scala
index 47c95fbc3ff..5b0c3fc6661 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveBroadcastJoinITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveJoinITCase.scala
@@ -20,7 +20,8 @@ package org.apache.flink.table.planner.runtime.batch.sql.adaptive
import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{LONG_TYPE_INFO, STRING_TYPE_INFO}
import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions}
-import org.apache.flink.table.planner.runtime.utils.BatchTestBase
+import org.apache.flink.table.planner.runtime.utils.AdaptiveBatchTestBase
+import org.apache.flink.table.planner.runtime.utils.AdaptiveBatchTestBase.row
import org.apache.flink.types.Row
import org.junit.jupiter.api.{BeforeEach, Test}
@@ -28,37 +29,12 @@ import org.junit.jupiter.api.{BeforeEach, Test}
import scala.collection.JavaConversions._
import scala.util.Random
-/** IT cases for adaptive broadcast join. */
-class AdaptiveBroadcastJoinITCase extends BatchTestBase {
+/** IT cases for adaptive join. */
+trait AdaptiveJoinITCase extends AdaptiveBatchTestBase {
@BeforeEach
override def before(): Unit = {
super.before()
-
- registerCollection(
- "T",
- AdaptiveBroadcastJoinITCase.dataT,
- AdaptiveBroadcastJoinITCase.rowType,
- "a, b, c, d",
- AdaptiveBroadcastJoinITCase.nullables)
- registerCollection(
- "T1",
- AdaptiveBroadcastJoinITCase.dataT1,
- AdaptiveBroadcastJoinITCase.rowType,
- "a1, b1, c1, d1",
- AdaptiveBroadcastJoinITCase.nullables)
- registerCollection(
- "T2",
- AdaptiveBroadcastJoinITCase.dataT2,
- AdaptiveBroadcastJoinITCase.rowType,
- "a2, b2, c2, d2",
- AdaptiveBroadcastJoinITCase.nullables)
- registerCollection(
- "T3",
- AdaptiveBroadcastJoinITCase.dataT3,
- AdaptiveBroadcastJoinITCase.rowType,
- "a3, b3, c3, d3",
- AdaptiveBroadcastJoinITCase.nullables)
}
@Test
@@ -124,6 +100,7 @@ class AdaptiveBroadcastJoinITCase extends BatchTestBase {
checkResult(sql)
}
+ @Test
def testSimpleJoin(): Unit = {
// inner join
val sql1 = "SELECT * FROM T1, T2 WHERE a1 = a2"
@@ -146,34 +123,26 @@ class AdaptiveBroadcastJoinITCase extends BatchTestBase {
checkResult(sql5)
}
- def checkResult(sql: String): Unit = {
- tEnv.getConfig
- .set(
- OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
- OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE)
- val expected = executeQuery(sql)
- tEnv.getConfig
- .set(
- OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
- OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.AUTO)
- checkResult(sql, expected)
- tEnv.getConfig
- .set(
- OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
- OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.RUNTIME_ONLY)
- checkResult(sql, expected)
- }
+ def checkResult(sql: String): Unit
}
-object AdaptiveBroadcastJoinITCase {
+object AdaptiveJoinITCase {
+
+ def generateRandomData: Seq[Row] = {
+ generateRandomData(Random.nextInt(30), 0)
+ }
- def generateRandomData(): Seq[Row] = {
+ def generateRandomData(numRows: Int, skewedFactor: Double): Seq[Row] = {
val data = new java.util.ArrayList[Row]()
- val numRows = Random.nextInt(30)
lazy val strs = Seq("adaptive", "join", "itcase")
for (x <- 0 until numRows) {
- data.add(
- BatchTestBase.row(x.toLong, Random.nextLong(), strs(Random.nextInt(3)), Random.nextLong()))
+ // Used to generate skewed data, records with random number less than skewedFactor will have the same id.
+ val id = if (Random.nextDouble() < skewedFactor) {
+ 0
+ } else {
+ x
+ }
+ data.add(row(id.toLong, Random.nextLong(), strs(Random.nextInt(3)), Random.nextLong()))
}
data
}
@@ -181,9 +150,4 @@ object AdaptiveBroadcastJoinITCase {
lazy val rowType =
new RowTypeInfo(LONG_TYPE_INFO, LONG_TYPE_INFO, STRING_TYPE_INFO, LONG_TYPE_INFO)
lazy val nullables: Array[Boolean] = Array(true, true, true, true)
-
- lazy val dataT: Seq[Row] = generateRandomData()
- lazy val dataT1: Seq[Row] = generateRandomData()
- lazy val dataT2: Seq[Row] = generateRandomData()
- lazy val dataT3: Seq[Row] = generateRandomData()
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveSkewedJoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveSkewedJoinITCase.scala
new file mode 100644
index 00000000000..4a2e05b82d0
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/adaptive/AdaptiveSkewedJoinITCase.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.planner.runtime.batch.sql.adaptive
+
+import org.apache.flink.configuration.{BatchExecutionOptions, MemorySize}
+import org.apache.flink.table.api.config.OptimizerConfigOptions
+
+import org.junit.jupiter.api.{BeforeEach, Test}
+
+/** IT cases for adaptive skewed join. */
+class AdaptiveSkewedJoinITCase extends AdaptiveJoinITCase {
+ @BeforeEach
+ override def before(): Unit = {
+ super.before()
+
+ tEnv.getConfig.set(BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_ENABLED, Boolean.box(true))
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
+ OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE)
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_SKEWED_THRESHOLD,
+ MemorySize.parse("100k"))
+
+ registerCollection(
+ "T",
+ AdaptiveJoinITCase.generateRandomData(30, 0),
+ AdaptiveJoinITCase.rowType,
+ "a, b, c, d",
+ AdaptiveJoinITCase.nullables,
+ forceNonParallel = false
+ )
+ registerCollection(
+ "T1",
+ AdaptiveJoinITCase.generateRandomData(3000, 0.99),
+ AdaptiveJoinITCase.rowType,
+ "a1, b1, c1, d1",
+ AdaptiveJoinITCase.nullables,
+ forceNonParallel = false
+ )
+ registerCollection(
+ "T2",
+ AdaptiveJoinITCase.generateRandomData(30, 0),
+ AdaptiveJoinITCase.rowType,
+ "a2, b2, c2, d2",
+ AdaptiveJoinITCase.nullables,
+ forceNonParallel = false)
+ registerCollection(
+ "T3",
+ AdaptiveJoinITCase.generateRandomData(30, 0),
+ AdaptiveJoinITCase.rowType,
+ "a3, b3, c3, d3",
+ AdaptiveJoinITCase.nullables,
+ forceNonParallel = false)
+ }
+
+ @Test
+ def testJoinWithSkewedUnionInput(): Unit = {
+ val sql =
+ """
+ |SELECT * FROM
+ | (SELECT * FROM (SELECT a1 as a, b1 as b, c1 as c, d1 as d FROM T1) UNION ALL (SELECT a2 as a, b2 as b, c2 as c, d2 as d FROM T2)) Y
+ | LEFT JOIN T ON T.a = Y.a
+ |""".stripMargin
+ checkResult(sql)
+ }
+
+ override def checkResult(sql: String): Unit = {
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE)
+ val expected = executeQuery(sql)
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.AUTO)
+ checkResult(sql, expected)
+ tEnv.getConfig
+ .set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.FORCED
+ )
+ checkResult(sql, expected)
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/join/JoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/join/JoinITCase.scala
index 794cb558c74..ede825dfe80 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/join/JoinITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/join/JoinITCase.scala
@@ -207,6 +207,9 @@ class JoinITCase extends BatchTestBase {
tEnv.getConfig.set(
OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
OptimizerConfigOptions.AdaptiveBroadcastJoinStrategy.NONE)
+ tEnv.getConfig.set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE)
val sink = (new CollectRowTableSink).configure(Array("c"), Array(Types.STRING))
tEnv.asInstanceOf[TableEnvironmentInternal].registerTableSinkInternal("outputTable", sink)
val stmtSet = tEnv.createStatementSet()
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchTestBase.scala
new file mode 100644
index 00000000000..eddf30d1a7d
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/AdaptiveBatchTestBase.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.planner.runtime.utils
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.table.api.{EnvironmentSettings, TableConfig, TableEnvironment}
+import org.apache.flink.table.api.config.ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM
+import org.apache.flink.table.api.internal.TableEnvironmentImpl
+import org.apache.flink.table.planner.delegation.PlannerBase
+import org.apache.flink.table.planner.factories.TestValuesTableFactory
+import org.apache.flink.table.planner.runtime.utils.AdaptiveBatchAbstractTestBase.DEFAULT_PARALLELISM
+import org.apache.flink.table.planner.runtime.utils.BatchTableEnvUtil.parseFieldNames
+import org.apache.flink.table.planner.runtime.utils.BatchTestBase.{check, checkSame}
+import org.apache.flink.table.planner.utils.TestingTableEnvironment
+import org.apache.flink.types.Row
+
+import org.junit.jupiter.api.{AfterEach, BeforeEach}
+
+class AdaptiveBatchTestBase extends AdaptiveBatchAbstractTestBase {
+ protected var tEnv: TableEnvironment = _
+
+ @throws(classOf[Exception])
+ @BeforeEach
+ def setupEnv(): Unit = {
+ tEnv = TestingTableEnvironment
+ .create(
+ EnvironmentSettings.newInstance().inBatchMode().build(),
+ catalogManager = None,
+ TableConfig.getDefault)
+ tEnv
+ .asInstanceOf[TableEnvironmentImpl]
+ .getPlanner
+ .asInstanceOf[PlannerBase]
+ .getExecEnv
+ .getConfig
+ .enableObjectReuse()
+ }
+
+ @throws(classOf[Exception])
+ @BeforeEach
+ def before(): Unit = {}
+
+ @AfterEach
+ def after(): Unit = {
+ TestValuesTableFactory.clearAllData()
+ }
+
+ def registerCollection[T](
+ tableName: String,
+ data: Iterable[T],
+ typeInfo: TypeInformation[T],
+ fields: String,
+ fieldNullables: Array[Boolean],
+ forceNonParallel: Boolean = true): Unit = {
+ BatchTableEnvUtil.registerCollection(
+ tEnv,
+ tableName,
+ data,
+ typeInfo,
+ Some(parseFieldNames(fields)),
+ Option(fieldNullables),
+ None,
+ forceNonParallel)
+ }
+
+ def checkResult(sqlQuery: String, expectedResult: Seq[Row], isSorted: Boolean = false): Unit = {
+ check(sqlQuery, (result: Seq[Row]) => checkSame(expectedResult, result, isSorted), tEnv)
+ }
+
+ def executeQuery(sqlQuery: String): Seq[Row] = {
+ BatchTestBase.executeQuery(sqlQuery, tEnv)
+ }
+}
+
+object AdaptiveBatchTestBase {
+ def configForMiniCluster(tableConfig: TableConfig): Unit = {
+ tableConfig.set(TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, Int.box(DEFAULT_PARALLELISM))
+ }
+
+ def row(args: Any*): Row = {
+ BatchTestBase.row(args: _*)
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTableEnvUtil.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTableEnvUtil.scala
index e9bedb9db9d..714f517d6cf 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTableEnvUtil.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTableEnvUtil.scala
@@ -243,6 +243,8 @@ object BatchTableEnvUtil {
* The field isNullables attributes of data.
* @param statistic
* statistics of current Table
+ * @param forceNonParallel
+ * sets the operator with only one parallelism
* @tparam T
* The type of the [[Iterable]].
* @return
@@ -256,14 +258,17 @@ object BatchTableEnvUtil {
typeInfo: TypeInformation[T],
fields: Option[Array[Expression]],
fieldNullables: Option[Array[Boolean]],
- statistic: Option[FlinkStatistic]): Unit = {
+ statistic: Option[FlinkStatistic],
+ forceNonParallel: Boolean = true): Unit = {
val execEnv = getPlanner(tEnv).getExecEnv
val boundedStream = execEnv.createInput(
new CollectionInputFormat[T](
data.asJavaCollection,
typeInfo.createSerializer(execEnv.getConfig.getSerializerConfig)),
typeInfo)
- boundedStream.forceNonParallel()
+ if (forceNonParallel) {
+ boundedStream.forceNonParallel()
+ }
registerBoundedStreamInternal(tEnv, tableName, boundedStream, fields, fieldNullables, statistic)
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala
index cb509321f34..5bf2b6eb3c3 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala
@@ -35,6 +35,7 @@ import org.apache.flink.table.planner.factories.TestValuesTableFactory
import org.apache.flink.table.planner.plan.stats.FlinkStatistic
import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil
import org.apache.flink.table.planner.runtime.utils.BatchAbstractTestBase.DEFAULT_PARALLELISM
+import org.apache.flink.table.planner.runtime.utils.BatchTestBase._
import org.apache.flink.table.planner.utils.{TableTestUtil, TestingTableEnvironment}
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo
import org.apache.flink.table.runtime.util.RowDataTestUtil
@@ -91,29 +92,8 @@ class BatchTestBase extends BatchAbstractTestBase {
TestValuesTableFactory.clearAllData()
}
- /**
- * Explain ast tree nodes of table and the logical plan after optimization.
- *
- * @param table
- * table to explain for
- * @return
- * string presentation of of explaining
- */
- def explainLogical(table: Table): String = {
- val ast = TableTestUtil.toRelNode(table)
- val logicalPlan = getPlan(ast)
-
- s"== Abstract Syntax Tree ==" +
- System.lineSeparator +
- s"${FlinkRelOptUtil.toString(ast)}" +
- System.lineSeparator +
- s"== Optimized Logical Plan ==" +
- System.lineSeparator +
- s"$logicalPlan"
- }
-
def checkResult(sqlQuery: String, expectedResult: Seq[Row], isSorted: Boolean = false): Unit = {
- check(sqlQuery, (result: Seq[Row]) => checkSame(expectedResult, result, isSorted))
+ check(sqlQuery, (result: Seq[Row]) => checkSame(expectedResult, result, isSorted), tEnv)
}
def checkTableResult(table: Table, expectedResult: Seq[Row], isSorted: Boolean = false): Unit = {
@@ -137,39 +117,17 @@ class BatchTestBase extends BatchAbstractTestBase {
""".stripMargin
Some(errorMessage)
} else None
- }
+ },
+ tEnv
)
}
- private def getPlan(relNode: RelNode): String = {
- val planner = tEnv.asInstanceOf[TableEnvironmentImpl].getPlanner.asInstanceOf[PlannerBase]
- val optimized = planner.optimize(relNode)
- FlinkRelOptUtil.toString(optimized, SqlExplainLevel.EXPPLAN_ATTRIBUTES)
- }
-
- def check(sqlQuery: String, checkFunc: Seq[Row] => Option[String]): Unit = {
- val table = parseQuery(sqlQuery)
- val result = executeQuery(table)
-
- checkFunc(result).foreach {
- results =>
- val plan = explainLogical(table)
- fail(s"""
- |Results do not match for query:
- | $sqlQuery
- |$results
- |Plan:
- | $plan
- """.stripMargin)
- }
- }
-
def checkTable(table: Table, checkFunc: Seq[Row] => Option[String]): Unit = {
val result = executeQuery(table)
checkFunc(result).foreach {
results =>
- val plan = explainLogical(table)
+ val plan = explainLogical(table, tEnv)
fail(s"""
|Results do not match:
|$results
@@ -310,10 +268,12 @@ class BatchTestBase extends BatchAbstractTestBase {
}
}
- def parseQuery(sqlQuery: String): Table = tEnv.sqlQuery(sqlQuery)
+ def parseQuery(sqlQuery: String): Table = {
+ BatchTestBase.parseQuery(sqlQuery, tEnv)
+ }
def executeQuery(table: Table): Seq[Row] = {
- CollectionUtil.iteratorToList(table.execute().collect()).asScala
+ BatchTestBase.executeQuery(table)
}
def executeQuery(sqlQuery: String): Seq[Row] = {
@@ -325,45 +285,11 @@ class BatchTestBase extends BatchAbstractTestBase {
if (!isSorted) seq.map(_.toString).sortBy(s => s) else seq.map(_.toString)
}
- def checkSame(
- expectedResult: Seq[Row],
- result: Seq[Row],
- isSorted: Boolean = false): Option[String] = {
- if (
- expectedResult.size != result.size
- || !prepareResult(expectedResult, isSorted).equals(prepareResult(result, isSorted))
- ) {
- val errorMessage =
- s"""
- |Results
- |${sideBySide(
- s"== Correct Result - ${expectedResult.size} ==" +:
- prepareResult(expectedResult, isSorted),
- s"== Actual Result - ${result.size} ==" +:
- prepareResult(result, isSorted)
- ).mkString("\n")}
- """.stripMargin
- Some(errorMessage)
- } else None
- }
-
private def checkEmpty(result: Seq[Row]) = {
val expectedResult = Nil
checkSame(expectedResult, result, isSorted = true)
}
- private def sideBySide(left: Seq[String], right: Seq[String]) = {
- val maxLeftSize = left.map(_.length).max
- val leftPadded = left ++ Seq.fill(math.max(right.size - left.size, 0))("")
- val rightPadded = right ++ Seq.fill(math.max(left.size - right.size, 0))("")
-
- leftPadded.zip(rightPadded).map {
- case (l, r) =>
- (if (l == r || l.startsWith("== Correct")) " " else "!") +
- l + (" " * ((maxLeftSize - l.length) + 3)) + r
- }
- }
-
implicit def registerCollection(
tableName: String,
data: Iterable[Row],
@@ -543,4 +469,110 @@ object BatchTestBase {
tableConfig.set(JobManagerOptions.SCHEDULER, JobManagerOptions.SchedulerType.AdaptiveBatch)
}
}
+
+ def checkResult(
+ sqlQuery: String,
+ expectedResult: Seq[Row],
+ tEnv: TableEnvironment,
+ isSorted: Boolean = false): Unit = {
+ check(sqlQuery, (result: Seq[Row]) => checkSame(expectedResult, result, isSorted), tEnv)
+ }
+
+ def check(
+ sqlQuery: String,
+ checkFunc: Seq[Row] => Option[String],
+ tEnv: TableEnvironment): Unit = {
+ val table = parseQuery(sqlQuery, tEnv)
+ val result = executeQuery(table)
+
+ checkFunc(result).foreach {
+ results =>
+ val plan = explainLogical(table, tEnv)
+ fail(s"""
+ |Results do not match for query:
+ | $sqlQuery
+ |$results
+ |Plan:
+ | $plan
+ """.stripMargin)
+ }
+ }
+
+ def parseQuery(sqlQuery: String, tEnv: TableEnvironment): Table = tEnv.sqlQuery(sqlQuery)
+
+ def executeQuery(table: Table): Seq[Row] = {
+ CollectionUtil.iteratorToList(table.execute().collect()).asScala
+ }
+
+ def executeQuery(sqlQuery: String, tEnv: TableEnvironment): Seq[Row] = {
+ val table = parseQuery(sqlQuery, tEnv)
+ executeQuery(table)
+ }
+
+ /**
+ * Explain ast tree nodes of table and the logical plan after optimization.
+ *
+ * @param table
+ * table to explain for
+ * @param tEnv
+ * table environment
+ * @return
+ * string presentation of explaining
+ */
+ def explainLogical(table: Table, tEnv: TableEnvironment): String = {
+ val ast = TableTestUtil.toRelNode(table)
+ val logicalPlan = getPlan(ast, tEnv)
+
+ s"== Abstract Syntax Tree ==" +
+ System.lineSeparator +
+ s"${FlinkRelOptUtil.toString(ast)}" +
+ System.lineSeparator +
+ s"== Optimized Logical Plan ==" +
+ System.lineSeparator +
+ s"$logicalPlan"
+ }
+
+ private def getPlan(relNode: RelNode, tEnv: TableEnvironment): String = {
+ val planner = tEnv.asInstanceOf[TableEnvironmentImpl].getPlanner.asInstanceOf[PlannerBase]
+ val optimized = planner.optimize(relNode)
+ FlinkRelOptUtil.toString(optimized, SqlExplainLevel.EXPPLAN_ATTRIBUTES)
+ }
+
+ def checkSame(
+ expectedResult: Seq[Row],
+ result: Seq[Row],
+ isSorted: Boolean = false): Option[String] = {
+ if (
+ expectedResult.size != result.size
+ || !prepareResult(expectedResult, isSorted).equals(prepareResult(result, isSorted))
+ ) {
+ val errorMessage =
+ s"""
+ |Results
+ |${sideBySide(
+ s"== Correct Result - ${expectedResult.size} ==" +:
+ prepareResult(expectedResult, isSorted),
+ s"== Actual Result - ${result.size} ==" +:
+ prepareResult(result, isSorted)
+ ).mkString("\n")}
+ """.stripMargin
+ Some(errorMessage)
+ } else None
+ }
+
+ private def prepareResult(seq: Seq[Row], isSorted: Boolean): Seq[String] = {
+ if (!isSorted) seq.map(_.toString).sortBy(s => s) else seq.map(_.toString)
+ }
+
+ private def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = {
+ val maxLeftSize = left.map(_.length).max
+ val leftPadded = left ++ Seq.fill(math.max(right.size - left.size, 0))("")
+ val rightPadded = right ++ Seq.fill(math.max(left.size - right.size, 0))("")
+
+ leftPadded.zip(rightPadded).map {
+ case (l, r) =>
+ (if (l == r || l.startsWith("== Correct")) " " else "!") +
+ l + (" " * ((maxLeftSize - l.length) + 3)) + r
+ }
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala
index 4da7813a81e..5f0ea736097 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala
@@ -1333,6 +1333,9 @@ abstract class TableTestUtil(
tableEnv.getConfig.set(
OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_BROADCAST_JOIN_STRATEGY,
NONE)
+ tableEnv.getConfig.set(
+ OptimizerConfigOptions.TABLE_OPTIMIZER_ADAPTIVE_SKEWED_JOIN_OPTIMIZATION_STRATEGY,
+ OptimizerConfigOptions.AdaptiveSkewedJoinOptimizationStrategy.NONE)
private val env: StreamExecutionEnvironment = getPlanner.getExecEnv
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveBroadcastJoinOptimizationStrategy.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveBroadcastJoinOptimizationStrategy.java
index 1c33ed9fb60..0f90a71762f 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveBroadcastJoinOptimizationStrategy.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveBroadcastJoinOptimizationStrategy.java
@@ -40,8 +40,10 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
+import static org.apache.flink.table.runtime.strategy.AdaptiveJoinOptimizationUtils.filterEdges;
+import static org.apache.flink.table.runtime.strategy.AdaptiveJoinOptimizationUtils.isBroadcastJoin;
+import static org.apache.flink.table.runtime.strategy.AdaptiveJoinOptimizationUtils.isBroadcastJoinDisabled;
import static org.apache.flink.util.Preconditions.checkState;
/** The stream graph optimization strategy of adaptive broadcast join. */
@@ -77,6 +79,10 @@ public class AdaptiveBroadcastJoinOptimizationStrategy
ImmutableStreamNode adaptiveJoinNode,
List<ImmutableStreamEdge> upstreamStreamEdges,
AdaptiveJoin adaptiveJoin) {
+ if (!canPerformOptimization(
+ adaptiveJoinNode, context.getStreamGraph().getConfiguration())) {
+ return;
+ }
for (ImmutableStreamEdge upstreamEdge : upstreamStreamEdges) {
IntermediateDataSetID relatedDataSetId =
context.getConsumedIntermediateDataSetId(upstreamEdge.getEdgeId());
@@ -165,6 +171,11 @@ public class AdaptiveBroadcastJoinOptimizationStrategy
}
}
+ private boolean canPerformOptimization(
+ ImmutableStreamNode adaptiveJoinNode, ReadableConfig config) {
+ return !isBroadcastJoinDisabled(config) && !isBroadcastJoin(adaptiveJoinNode);
+ }
+
private void aggregatedInputBytesByTypeNumber(
ImmutableStreamNode adaptiveJoinNode, int typeNumber, long producedBytes) {
Integer streamNodeId = adaptiveJoinNode.getId();
@@ -174,13 +185,6 @@ public class AdaptiveBroadcastJoinOptimizationStrategy
.merge(typeNumber, producedBytes, Long::sum);
}
- private List<ImmutableStreamEdge> filterEdges(
- List<ImmutableStreamEdge> inEdges, int typeNumber) {
- return inEdges.stream()
- .filter(e -> e.getTypeNumber() == typeNumber)
- .collect(Collectors.toList());
- }
-
private List<StreamEdgeUpdateRequestInfo> generateStreamEdgeUpdateRequestInfos(
List<ImmutableStreamEdge> modifiedEdges, StreamPartitioner<?> outputPartitioner) {
List<StreamEdgeUpdateRequestInfo> streamEdgeUpdateRequestInfos = new ArrayList<>();
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveJoinOptimizationUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveJoinOptimizationUtils.java
new file mode 100644
index 00000000000..d0c637d0527
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/strategy/AdaptiveJoinOptimizationUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.strategy;
+
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.streaming.api.graph.util.ImmutableStreamEdge;
+import org.apache.flink.streaming.api.graph.util.ImmutableStreamNode;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.api.config.ExecutionConfigOptions.TABLE_EXEC_DISABLED_OPERATORS;
+
+/** Utils for adaptive join optimization strategies. */
+public class AdaptiveJoinOptimizationUtils {
+
+ public static List<ImmutableStreamEdge> filterEdges(
+ List<ImmutableStreamEdge> inEdges, int typeNumber) {
+ return inEdges.stream()
+ .filter(e -> e.getTypeNumber() == typeNumber)
+ .collect(Collectors.toList());
+ }
+
+ public static boolean isBroadcastJoin(ImmutableStreamNode adaptiveJoinNode) {
+ return adaptiveJoinNode.getInEdges().stream()
+ .anyMatch(ImmutableStreamEdge::isBroadcastEdge);
+ }
+
+ public static boolean isBroadcastJoinDisabled(ReadableConfig config) {
+ String value = config.get(TABLE_EXEC_DISABLED_OPERATORS);
+ if (value == null) {
+ return false;
+ }
+ String[] operators = value.split(",");
+ for (String operator : operators) {
+ operator = operator.trim();
+ if (operator.isEmpty()) {
+ continue;
+ }
+ if (operator.equals("HashJoin") || operator.equals("BroadcastHashJoin")) {
+ return true;
+ }
+ }
+ return false;
+ }
+}