You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by wa...@apache.org on 2023/02/14 13:34:30 UTC
[flink] branch FLINK-31055 created (now c261abaf64d)
This is an automated email from the ASF dual-hosted git repository.
wanglijie pushed a change to branch FLINK-31055
in repository https://gitbox.apache.org/repos/asf/flink.git
at c261abaf64d [FLINK-31055][runtime] Fix the bug that dynamic flag of stream graph does not take effect when translating the transformations
This branch includes the following new commits:
new 4996996a3f5 [hotfix] Fix the missing parameter error in ForwardForConsecutiveHashPartitionerTest
new c261abaf64d [FLINK-31055][runtime] Fix the bug that dynamic flag of stream graph does not take effect when translating the transformations
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[flink] 01/02: [hotfix] Fix the missing parameter error in ForwardForConsecutiveHashPartitionerTest
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wanglijie pushed a commit to branch FLINK-31055
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4996996a3f5caacf79911af87c538085ac18c9d0
Author: Lijie Wang <wa...@gmail.com>
AuthorDate: Tue Feb 14 21:32:51 2023 +0800
[hotfix] Fix the missing parameter error in ForwardForConsecutiveHashPartitionerTest
---
.../runtime/partitioner/ForwardForConsecutiveHashPartitionerTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForConsecutiveHashPartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForConsecutiveHashPartitionerTest.java
index c33adbd92a8..99e2f10a610 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForConsecutiveHashPartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForConsecutiveHashPartitionerTest.java
@@ -69,7 +69,8 @@ class ForwardForConsecutiveHashPartitionerTest extends TestLogger {
"group1",
"group2",
new ForwardForConsecutiveHashPartitioner<>(
- new KeyGroupStreamPartitioner<>(record -> 0L, 100)));
+ new KeyGroupStreamPartitioner<>(record -> 0L, 100)),
+ streamExchangeMode);
List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
Assertions.assertThat(jobVertices.size()).isEqualTo(2);
JobVertex sourceVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
[flink] 02/02: [FLINK-31055][runtime] Fix the bug that dynamic flag of stream graph does not take effect when translating the transformations
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wanglijie pushed a commit to branch FLINK-31055
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c261abaf64d094110e4c35494c77727e885c3378
Author: Lijie Wang <wa...@gmail.com>
AuthorDate: Tue Feb 14 21:33:54 2023 +0800
[FLINK-31055][runtime] Fix the bug that dynamic flag of stream graph does not take effect when translating the transformations
---
.../streaming/api/graph/StreamGraphGenerator.java | 17 ++++---
.../ForwardForUnspecifiedPartitionerTest.java | 52 +++++++++++++++++++---
2 files changed, 55 insertions(+), 14 deletions(-)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index ba05b6b9c22..287d18aca8f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -316,6 +316,14 @@ public class StreamGraphGenerator {
configuration.get(
ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH));
shouldExecuteInBatchMode = shouldExecuteInBatchMode();
+ Optional<JobManagerOptions.SchedulerType> schedulerTypeOptional =
+ executionConfig.getSchedulerType();
+ boolean dynamic =
+ shouldExecuteInBatchMode
+ && schedulerTypeOptional.orElse(
+ JobManagerOptions.SchedulerType.AdaptiveBatch)
+ == JobManagerOptions.SchedulerType.AdaptiveBatch;
+ streamGraph.setDynamic(dynamic);
configureStreamGraph(streamGraph);
alreadyTransformed = new IdentityHashMap<>();
@@ -328,15 +336,6 @@ public class StreamGraphGenerator {
setFineGrainedGlobalStreamExchangeMode(streamGraph);
- Optional<JobManagerOptions.SchedulerType> schedulerTypeOptional =
- executionConfig.getSchedulerType();
- boolean dynamic =
- shouldExecuteInBatchMode
- && schedulerTypeOptional.orElse(
- JobManagerOptions.SchedulerType.AdaptiveBatch)
- == JobManagerOptions.SchedulerType.AdaptiveBatch;
- streamGraph.setDynamic(dynamic);
-
for (StreamNode node : streamGraph.getStreamNodes()) {
if (node.getInEdges().stream().anyMatch(this::shouldDisableUnalignedCheckpointing)) {
for (StreamEdge edge : node.getInEdges()) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForUnspecifiedPartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForUnspecifiedPartitionerTest.java
index 1f736ad19de..648c26949be 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForUnspecifiedPartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/ForwardForUnspecifiedPartitionerTest.java
@@ -17,18 +17,24 @@
package org.apache.flink.streaming.runtime.partitioner;
+import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.api.graph.NonChainedOutput;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.graph.StreamEdge;
import org.apache.flink.util.TestLogger;
-import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Test for {@link ForwardForUnspecifiedPartitioner}. */
class ForwardForUnspecifiedPartitionerTest extends TestLogger {
@@ -38,12 +44,12 @@ class ForwardForUnspecifiedPartitionerTest extends TestLogger {
StreamPartitionerTestUtils.createJobGraph(
"group1", "group1", new ForwardForUnspecifiedPartitioner<>());
List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
- Assertions.assertThat(jobVertices.size()).isEqualTo(1);
+ assertThat(jobVertices.size()).isEqualTo(1);
JobVertex vertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
StreamConfig sourceConfig = new StreamConfig(vertex.getConfiguration());
StreamEdge edge = sourceConfig.getChainedOutputs(getClass().getClassLoader()).get(0);
- Assertions.assertThat(edge.getPartitioner()).isInstanceOf(ForwardPartitioner.class);
+ assertThat(edge.getPartitioner()).isInstanceOf(ForwardPartitioner.class);
}
@Test
@@ -52,12 +58,48 @@ class ForwardForUnspecifiedPartitionerTest extends TestLogger {
StreamPartitionerTestUtils.createJobGraph(
"group1", "group2", new ForwardForUnspecifiedPartitioner<>());
List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
- Assertions.assertThat(jobVertices.size()).isEqualTo(2);
+ assertThat(jobVertices.size()).isEqualTo(2);
JobVertex sourceVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
NonChainedOutput output =
sourceConfig.getOperatorNonChainedOutputs(getClass().getClassLoader()).get(0);
- Assertions.assertThat(output.getPartitioner()).isInstanceOf(RescalePartitioner.class);
+ assertThat(output.getPartitioner()).isInstanceOf(RescalePartitioner.class);
+ }
+
+ @Test
+ void testConvertToCorrectPartitioner() {
+ testConvertToCorrectPartitioner(null, RescalePartitioner.class);
+ testConvertToCorrectPartitioner(
+ JobManagerOptions.SchedulerType.AdaptiveBatch, RescalePartitioner.class);
+ testConvertToCorrectPartitioner(
+ JobManagerOptions.SchedulerType.Default, ForwardPartitioner.class);
+ }
+
+ private void testConvertToCorrectPartitioner(
+ JobManagerOptions.SchedulerType scheduler, Class<?> expectedPartitioner) {
+ Configuration configuration = new Configuration();
+ if (scheduler != null) {
+ configuration.set(JobManagerOptions.SCHEDULER, scheduler);
+ }
+ final StreamExecutionEnvironment env =
+ StreamExecutionEnvironment.getExecutionEnvironment(configuration);
+ env.setRuntimeMode(RuntimeExecutionMode.BATCH);
+ env.fromSequence(0, 99)
+ .slotSharingGroup("group1")
+ .name("source")
+ .addSink(new DiscardingSink<>())
+ .slotSharingGroup("group2")
+ .name("sink");
+
+ List<JobVertex> jobVertices =
+ env.getStreamGraph().getJobGraph().getVerticesSortedTopologicallyFromSources();
+ assertThat(jobVertices.size()).isEqualTo(2);
+
+ JobVertex sourceVertex = jobVertices.get(0);
+ StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
+ NonChainedOutput output =
+ sourceConfig.getOperatorNonChainedOutputs(getClass().getClassLoader()).get(0);
+ assertThat(output.getPartitioner()).isInstanceOf(expectedPartitioner);
}
}