You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by yi...@apache.org on 2022/07/20 10:44:42 UTC
[flink] branch master updated: [FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service
This is an automated email from the ASF dual-hosted git repository.
yingjie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new e630778eb8b [FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service
e630778eb8b is described below
commit e630778eb8b500148b01583d34ec195a80c99c3f
Author: Yuxin Tan <ta...@gmail.com>
AuthorDate: Thu Jul 7 18:46:41 2022 +0800
[FLINK-28446][runtime] Expose more information in PartitionDescriptor to support more optimized Shuffle Service
This closes #20200.
---
.../flink/runtime/shuffle/PartitionDescriptor.java | 39 +++++++++++++++++++---
.../ResultPartitionDeploymentDescriptorTest.java | 6 +++-
.../runtime/shuffle/NettyShuffleUtilsTest.java | 4 ++-
.../shuffle/PartitionDescriptorBuilder.java | 4 ++-
4 files changed, 46 insertions(+), 7 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
index 0d60051054a..faed01cb23c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/PartitionDescriptor.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.executiongraph.IntermediateResult;
import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
@@ -53,6 +54,15 @@ public class PartitionDescriptor implements Serializable {
/** Connection index to identify this partition of intermediate result. */
private final int connectionIndex;
+ /** Whether the intermediate result is a broadcast result. */
+ private final boolean isBroadcast;
+
+ /**
+ * Whether the distribution pattern of the intermediate result is {@link
+ * DistributionPattern.ALL_TO_ALL}.
+ */
+ private final boolean isAllToAllDistribution;
+
@VisibleForTesting
public PartitionDescriptor(
IntermediateDataSetID resultId,
@@ -60,7 +70,9 @@ public class PartitionDescriptor implements Serializable {
IntermediateResultPartitionID partitionId,
ResultPartitionType partitionType,
int numberOfSubpartitions,
- int connectionIndex) {
+ int connectionIndex,
+ boolean isBroadcast,
+ boolean isAllToAllDistribution) {
this.resultId = checkNotNull(resultId);
checkArgument(totalNumberOfPartitions >= 1);
this.totalNumberOfPartitions = totalNumberOfPartitions;
@@ -69,6 +81,8 @@ public class PartitionDescriptor implements Serializable {
checkArgument(numberOfSubpartitions >= 1);
this.numberOfSubpartitions = numberOfSubpartitions;
this.connectionIndex = connectionIndex;
+ this.isBroadcast = isBroadcast;
+ this.isAllToAllDistribution = isAllToAllDistribution;
}
public IntermediateDataSetID getResultId() {
@@ -95,12 +109,27 @@ public class PartitionDescriptor implements Serializable {
return connectionIndex;
}
+ public boolean isBroadcast() {
+ return isBroadcast;
+ }
+
+ public boolean isAllToAllDistribution() {
+ return isAllToAllDistribution;
+ }
+
@Override
public String toString() {
return String.format(
"PartitionDescriptor [result id: %s, partition id: %s, partition type: %s, "
- + "subpartitions: %d, connection index: %d]",
- resultId, partitionId, partitionType, numberOfSubpartitions, connectionIndex);
+ + "subpartitions: %d, connection index: %d, is broadcast: %s, "
+ + "is all-to-all distribution: %s]",
+ resultId,
+ partitionId,
+ partitionType,
+ numberOfSubpartitions,
+ connectionIndex,
+ isBroadcast,
+ isAllToAllDistribution);
}
public static PartitionDescriptor from(IntermediateResultPartition partition) {
@@ -113,6 +142,8 @@ public class PartitionDescriptor implements Serializable {
partition.getPartitionId(),
result.getResultType(),
partition.getNumberOfSubpartitions(),
- result.getConnectionIndex());
+ result.getConnectionIndex(),
+ result.isBroadcast(),
+ result.getConsumingDistributionPattern() == DistributionPattern.ALL_TO_ALL);
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
index 9f0fd1a9dfe..33c79a1be4a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/ResultPartitionDeploymentDescriptorTest.java
@@ -55,6 +55,8 @@ public class ResultPartitionDeploymentDescriptorTest extends TestLogger {
private static final ResultPartitionType partitionType = ResultPartitionType.PIPELINED;
private static final int numberOfSubpartitions = 24;
private static final int connectionIndex = 10;
+ private static final boolean isBroadcast = false;
+ private static final boolean isAllToAllDistribution = true;
private static final PartitionDescriptor partitionDescriptor =
new PartitionDescriptor(
@@ -63,7 +65,9 @@ public class ResultPartitionDeploymentDescriptorTest extends TestLogger {
partitionId,
partitionType,
numberOfSubpartitions,
- connectionIndex);
+ connectionIndex,
+ isBroadcast,
+ isAllToAllDistribution);
private static final ResultPartitionID resultPartitionID =
new ResultPartitionID(partitionId, producerExecutionId);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
index 64c74b248eb..748b26cbc3c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/NettyShuffleUtilsTest.java
@@ -174,7 +174,9 @@ public class NettyShuffleUtilsTest extends TestLogger {
shuffleDescriptor.getResultPartitionID().getPartitionId(),
resultPartitionType,
numSubpartitions,
- 0);
+ 0,
+ false,
+ true);
ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor =
new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
index a401f56770a..de85cd10adc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/shuffle/PartitionDescriptorBuilder.java
@@ -55,7 +55,9 @@ public class PartitionDescriptorBuilder {
partitionId,
partitionType,
1,
- 0);
+ 0,
+ false,
+ true);
}
public static PartitionDescriptorBuilder newBuilder() {