You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by yi...@apache.org on 2022/07/20 10:44:42 UTC

[flink] branch master updated: [FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service

This is an automated email from the ASF dual-hosted git repository.

yingjie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new e630778eb8b [FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service
e630778eb8b is described below

commit e630778eb8b500148b01583d34ec195a80c99c3f
Author: Yuxin Tan <ta...@gmail.com>
AuthorDate: Thu Jul 7 18:46:41 2022 +0800

    [FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service
    
    This closes #20200.
---
 .../flink/runtime/shuffle/PartitionDescriptor.java | 39 +++++++++++++++++++---
 .../ResultPartitionDeploymentDescriptorTest.java   |  6 +++-
 .../runtime/shuffle/NettyShuffleUtilsTest.java     |  4 ++-
 .../shuffle/PartitionDescriptorBuilder.java        |  4 ++-
 4 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
index 0d60051054a..faed01cb23c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.runtime.executiongraph.IntermediateResult;
 import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
 import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
 
@@ -53,6 +54,15 @@ public class PartitionDescriptor implements Serializable {
     /** Connection index to identify this partition of intermediate result. */
     private final int connectionIndex;
 
+    /** Whether the intermediate result is a broadcast result. */
+    private final boolean isBroadcast;
+
+    /**
+     * Whether the distribution pattern of the intermediate result is {@link
+     * DistributionPattern.ALL_TO_ALL}.
+     */
+    private final boolean isAllToAllDistribution;
+
     @VisibleForTesting
     public PartitionDescriptor(
             IntermediateDataSetID resultId,
@@ -60,7 +70,9 @@ public class PartitionDescriptor implements Serializable {
             IntermediateResultPartitionID partitionId,
             ResultPartitionType partitionType,
             int numberOfSubpartitions,
-            int connectionIndex) {
+            int connectionIndex,
+            boolean isBroadcast,
+            boolean isAllToAllDistribution) {
         this.resultId = checkNotNull(resultId);
         checkArgument(totalNumberOfPartitions >= 1);
         this.totalNumberOfPartitions = totalNumberOfPartitions;
@@ -69,6 +81,8 @@ public class PartitionDescriptor implements Serializable {
         checkArgument(numberOfSubpartitions >= 1);
         this.numberOfSubpartitions = numberOfSubpartitions;
         this.connectionIndex = connectionIndex;
+        this.isBroadcast = isBroadcast;
+        this.isAllToAllDistribution = isAllToAllDistribution;
     }
 
     public IntermediateDataSetID getResultId() {
@@ -95,12 +109,27 @@ public class PartitionDescriptor implements Serializable {
         return connectionIndex;
     }
 
+    public boolean isBroadcast() {
+        return isBroadcast;
+    }
+
+    public boolean isAllToAllDistribution() {
+        return isAllToAllDistribution;
+    }
+
     @Override
     public String toString() {
         return String.format(
                 "PartitionDescriptor [result id: %s, partition id: %s, partition type: %s, "
-                        + "subpartitions: %d, connection index: %d]",
-                resultId, partitionId, partitionType, numberOfSubpartitions, connectionIndex);
+                        + "subpartitions: %d, connection index: %d, is broadcast: %s, "
+                        + "is all-to-all distribution: %s]",
+                resultId,
+                partitionId,
+                partitionType,
+                numberOfSubpartitions,
+                connectionIndex,
+                isBroadcast,
+                isAllToAllDistribution);
     }
 
     public static PartitionDescriptor from(IntermediateResultPartition partition) {
@@ -113,6 +142,8 @@ public class PartitionDescriptor implements Serializable {
                 partition.getPartitionId(),
                 result.getResultType(),
                 partition.getNumberOfSubpartitions(),
-                result.getConnectionIndex());
+                result.getConnectionIndex(),
+                result.isBroadcast(),
+                result.getConsumingDistributionPattern() == DistributionPattern.ALL_TO_ALL);
     }
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
index 9f0fd1a9dfe..33c79a1be4a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
@@ -55,6 +55,8 @@ public class ResultPartitionDeploymentDescriptorTest extends TestLogger {
     private static final ResultPartitionType partitionType = ResultPartitionType.PIPELINED;
     private static final int numberOfSubpartitions = 24;
     private static final int connectionIndex = 10;
+    private static final boolean isBroadcast = false;
+    private static final boolean isAllToAllDistribution = true;
 
     private static final PartitionDescriptor partitionDescriptor =
             new PartitionDescriptor(
@@ -63,7 +65,9 @@ public class ResultPartitionDeploymentDescriptorTest extends TestLogger {
                     partitionId,
                     partitionType,
                     numberOfSubpartitions,
-                    connectionIndex);
+                    connectionIndex,
+                    isBroadcast,
+                    isAllToAllDistribution);
 
     private static final ResultPartitionID resultPartitionID =
             new ResultPartitionID(partitionId, producerExecutionId);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
index 64c74b248eb..748b26cbc3c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
@@ -174,7 +174,9 @@ public class NettyShuffleUtilsTest extends TestLogger {
                         shuffleDescriptor.getResultPartitionID().getPartitionId(),
                         resultPartitionType,
                         numSubpartitions,
-                        0);
+                        0,
+                        false,
+                        true);
         ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor =
                 new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
index a401f56770a..de85cd10adc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
@@ -55,7 +55,9 @@ public class PartitionDescriptorBuilder {
                 partitionId,
                 partitionType,
                 1,
-                0);
+                0,
+                false,
+                true);
     }
 
     public static PartitionDescriptorBuilder newBuilder() {