You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2024/06/24 07:42:14 UTC
(flink) 04/06: Revert "[FLINK-33668][runtime] Decouple the network memory and job topology on result partition side"
This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 64c54e8776a3addb7ed9ba103ccadebce1a6c5e4
Author: Yuxin Tan <ta...@gmail.com>
AuthorDate: Fri Jun 21 14:00:06 2024 +0800
Revert "[FLINK-33668][runtime] Decouple the network memory and job topology on result partition side"
This reverts commit 88138d08e731d3084d59bf14cc2fb51bdf4afbd8.
---
.../generated/all_taskmanager_network_section.html | 6 -
.../netty_shuffle_environment_configuration.html | 6 -
.../NettyShuffleEnvironmentOptions.java | 13 -
.../io/network/NettyShuffleServiceFactory.java | 1 -
.../network/partition/ResultPartitionFactory.java | 8 -
.../runtime/io/network/partition/SortBuffer.java | 4 +-
.../network/partition/consumer/BufferManager.java | 2 +-
.../partition/consumer/InputGateSpecUtils.java | 8 +-
.../partition/consumer/SingleInputGateFactory.java | 6 +-
.../shuffle/TieredResultPartitionFactory.java | 7 +-
.../tiered/storage/SortBufferAccumulator.java | 68 +---
.../tiered/storage/TieredStorageMemoryManager.java | 3 -
.../storage/TieredStorageMemoryManagerImpl.java | 63 ++--
.../flink/runtime/shuffle/NettyShuffleUtils.java | 37 +-
.../NettyShuffleEnvironmentConfiguration.java | 13 -
.../io/network/NettyShuffleEnvironmentBuilder.java | 9 -
.../io/network/buffer/TestingBufferPool.java | 419 ++-------------------
...editBasedPartitionRequestClientHandlerTest.java | 7 +-
.../NettyMessageClientDecoderDelegateTest.java | 6 +-
.../NettyMessageClientSideSerializationTest.java | 6 +-
.../network/partition/InputGateFairnessTest.java | 9 +-
.../network/partition/ResultPartitionBuilder.java | 8 -
.../partition/ResultPartitionFactoryTest.java | 1 -
.../partition/consumer/GateBuffersSpecTest.java | 49 +--
.../partition/consumer/RemoteInputChannelTest.java | 21 +-
.../partition/consumer/SingleInputGateBuilder.java | 9 +-
.../tiered/TestingTieredStorageMemoryManager.java | 20 -
.../tiered/shuffle/TieredResultPartitionTest.java | 10 +-
.../tiered/storage/SortBufferAccumulatorTest.java | 15 +-
.../flink/test/runtime/BatchShuffleITCaseBase.java | 22 +-
.../HybridShuffleMemoryDecouplingITCase.java | 114 ------
31 files changed, 126 insertions(+), 844 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
index b0b957154ca..14e7532b050 100644
--- a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
+++ b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
@@ -26,12 +26,6 @@
<td>String</td>
<td>The option configures the class that is responsible for creating an external remote tier factory for hybrid shuffle. If configured, the hybrid shuffle will only initialize the specified remote tier according to the given class name. Currently, since the tier interfaces are not yet public and are still actively evolving, it is recommended that users do not independently implement the external remote tier until the tier interfaces are stabilized. </td>
</tr>
- <tr>
- <td><h5>taskmanager.network.hybrid-shuffle.memory-decoupling.enabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>The option is used to make the memory that is used by hybrid shuffle decoupled from the complexity of the job topology and the number of tasks on the task manager. It significantly reduces the chance of the "Insufficient number of network buffers" exception, while the workloads may suffer performance reduction silently.</td>
- </tr>
<tr>
<td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
index 1d3d33779a2..cfd54b05540 100644
--- a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
+++ b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
@@ -44,12 +44,6 @@
<td>String</td>
<td>The option configures the class that is responsible for creating an external remote tier factory for hybrid shuffle. If configured, the hybrid shuffle will only initialize the specified remote tier according to the given class name. Currently, since the tier interfaces are not yet public and are still actively evolving, it is recommended that users do not independently implement the external remote tier until the tier interfaces are stabilized. </td>
</tr>
- <tr>
- <td><h5>taskmanager.network.hybrid-shuffle.memory-decoupling.enabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>The option is used to make the memory that is used by hybrid shuffle decoupled from the complexity of the job topology and the number of tasks on the task manager. It significantly reduces the chance of the "Insufficient number of network buffers" exception, while the workloads may suffer performance reduction silently.</td>
- </tr>
<tr>
<td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
index 3f5c2b2e5cd..72cf1404cb4 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
@@ -525,19 +525,6 @@ public class NettyShuffleEnvironmentOptions {
+ HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
+ " is true, the remote storage will be disabled.");
- /** The option to decouple the needed memory of hybrid shuffle and the job topology. */
- @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
- @Experimental
- public static final ConfigOption<Boolean> NETWORK_HYBRID_SHUFFLE_ENABLE_MEMORY_DECOUPLING =
- key("taskmanager.network.hybrid-shuffle.memory-decoupling.enabled")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- "The option is used to make the memory that is used by hybrid shuffle decoupled from the complexity of the"
- + " job topology and the number of tasks on the task manager. It significantly reduces the chance of "
- + "the \"Insufficient number of network buffers\" exception, while the workloads may suffer performance "
- + "reduction silently.");
-
/**
* @deprecated The hash-based blocking shuffle is deprecated in 1.20 and will be totally removed
* in 2.0.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleServiceFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleServiceFactory.java
index f2e65d7317c..d8d068d9aee 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleServiceFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleServiceFactory.java
@@ -238,7 +238,6 @@ public class NettyShuffleServiceFactory
config.getMaxOverdraftBuffersPerGate(),
config.getHybridShuffleSpilledIndexRegionGroupSize(),
config.getHybridShuffleNumRetainedInMemoryRegionsMax(),
- config.isMemoryDecouplingEnabled(),
tieredResultPartitionFactory);
SingleInputGateFactory singleInputGateFactory =
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
index fa100455708..1bf21f8c7bc 100755
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
@@ -30,7 +30,6 @@ import org.apache.flink.runtime.io.network.buffer.BufferPool;
import org.apache.flink.runtime.io.network.buffer.BufferPoolFactory;
import org.apache.flink.runtime.io.network.partition.hybrid.HsResultPartition;
import org.apache.flink.runtime.io.network.partition.hybrid.HybridShuffleConfiguration;
-import org.apache.flink.runtime.io.network.partition.hybrid.tiered.common.TieredStorageUtils;
import org.apache.flink.runtime.io.network.partition.hybrid.tiered.shuffle.TieredResultPartitionFactory;
import org.apache.flink.runtime.io.network.partition.hybrid.tiered.storage.TieredStorageMemorySpec;
import org.apache.flink.runtime.io.network.partition.hybrid.tiered.tier.TierFactory;
@@ -90,8 +89,6 @@ public class ResultPartitionFactory {
private final long hybridShuffleNumRetainedInMemoryRegionsMax;
- private final boolean memoryDecouplingEnabled;
-
private final boolean sslEnabled;
private final int maxOverdraftBuffersPerGate;
@@ -117,7 +114,6 @@ public class ResultPartitionFactory {
int maxOverdraftBuffersPerGate,
int hybridShuffleSpilledIndexRegionGroupSize,
long hybridShuffleNumRetainedInMemoryRegionsMax,
- boolean memoryDecouplingEnabled,
Optional<TieredResultPartitionFactory> tieredStorage) {
this.partitionManager = partitionManager;
@@ -139,7 +135,6 @@ public class ResultPartitionFactory {
this.hybridShuffleSpilledIndexRegionGroupSize = hybridShuffleSpilledIndexRegionGroupSize;
this.hybridShuffleNumRetainedInMemoryRegionsMax =
hybridShuffleNumRetainedInMemoryRegionsMax;
- this.memoryDecouplingEnabled = memoryDecouplingEnabled;
this.tieredStorage = tieredStorage;
}
@@ -272,7 +267,6 @@ public class ResultPartitionFactory {
maxParallelism,
networkBufferSize,
isBroadcast,
- memoryDecouplingEnabled,
partitionManager,
bufferCompressor,
checkNotNull(
@@ -397,11 +391,9 @@ public class ResultPartitionFactory {
sortShuffleMinBuffers,
numberOfSubpartitions,
tieredStorage.isPresent(),
- memoryDecouplingEnabled,
tieredStorage
.map(ResultPartitionFactory::getNumTotalGuaranteedBuffers)
.orElse(0),
- TieredStorageUtils.getMinBuffersPerResultPartition(),
type);
return bufferPoolFactory.createBufferPool(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SortBuffer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SortBuffer.java
index 371484696a6..6bdd0eaa65a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SortBuffer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SortBuffer.java
@@ -242,11 +242,11 @@ public abstract class SortBuffer implements DataBuffer {
}
// allocate exactly enough buffers for the appended record
- while (availableBytes < numBytesRequired) {
+ do {
MemorySegment segment = freeSegments.poll();
availableBytes += bufferSize;
addBuffer(checkNotNull(segment));
- }
+ } while (availableBytes < numBytesRequired);
return true;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
index 8c32655a6cb..9a39e82168a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
@@ -174,7 +174,7 @@ public class BufferManager implements BufferListener, BufferRecycler {
/** Requests exclusive buffers from the local buffer pool. */
void requestExclusiveBuffers() {
synchronized (bufferQueue) {
- checkState(numExclusiveBuffers >= 0, "Num exclusive buffers must be non-negative.");
+ checkArgument(numExclusiveBuffers >= 0, "Num exclusive buffers must be non-negative.");
resizeBufferQueue();
if (numExclusiveBuffers == 0) {
return;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateSpecUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateSpecUtils.java
index 59ec4c5a903..661843cd096 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateSpecUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateSpecUtils.java
@@ -39,8 +39,7 @@ public class InputGateSpecUtils {
int configuredFloatingNetworkBuffersPerGate,
ResultPartitionType partitionType,
int numInputChannels,
- boolean enableTieredStorage,
- boolean memoryDecouplingEnabled) {
+ boolean enableTieredStorage) {
int maxRequiredBuffersThresholdPerGate =
getEffectiveMaxRequiredBuffersPerGate(
partitionType, configuredMaxRequiredBuffersPerGate, enableTieredStorage);
@@ -60,9 +59,7 @@ public class InputGateSpecUtils {
numInputChannels,
expectedBuffersPerGate);
int minBuffersPerGate =
- partitionType.isHybridResultPartition()
- && enableTieredStorage
- && memoryDecouplingEnabled
+ partitionType.isHybridResultPartition() && enableTieredStorage
? TieredStorageUtils.getMinBuffersPerGate()
: expectedBuffersPerGate;
expectedBuffersPerGate = Math.max(minBuffersPerGate, expectedBuffersPerGate);
@@ -115,6 +112,7 @@ public class InputGateSpecUtils {
return numInputChannels * configuredNetworkBuffersPerChannel + 1;
}
+ /** */
private static int getMaxBuffersPerGate(
int numInputChannels,
int configuredNetworkBuffersPerChannel,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateFactory.java
index 72b49b0a6e3..35700a2c460 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateFactory.java
@@ -104,8 +104,6 @@ public class SingleInputGateFactory {
private final int networkBufferSize;
- private final boolean memoryDecouplingEnabled;
-
private final BufferDebloatConfiguration debloatConfiguration;
/** The following attributes will be null if tiered storage shuffle is disabled. */
@@ -134,7 +132,6 @@ public class SingleInputGateFactory {
this.batchShuffleCompressionEnabled = networkConfig.isBatchShuffleCompressionEnabled();
this.compressionCodec = networkConfig.getCompressionCodec();
this.networkBufferSize = networkConfig.networkBufferSize();
- this.memoryDecouplingEnabled = networkConfig.isMemoryDecouplingEnabled();
this.connectionManager = connectionManager;
this.partitionManager = partitionManager;
this.taskEventPublisher = taskEventPublisher;
@@ -167,8 +164,7 @@ public class SingleInputGateFactory {
floatingNetworkBuffersPerGate,
igdd.getConsumedPartitionType(),
calculateNumNonLocalChannels(igdd, isSharedInputChannelSupported),
- tieredStorageConfiguration != null,
- memoryDecouplingEnabled);
+ tieredStorageConfiguration != null);
SupplierWithException<BufferPool, IOException> bufferPoolFactory =
createBufferPoolFactory(
networkBufferPool,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionFactory.java
index 8a788fe9258..4a00401c4d9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionFactory.java
@@ -93,7 +93,6 @@ public class TieredResultPartitionFactory {
int maxParallelism,
int bufferSizeBytes,
Boolean isBroadCastOnly,
- boolean memoryDecouplingEnabled,
ResultPartitionManager partitionManager,
@Nullable BufferCompressor bufferCompressor,
List<TierShuffleDescriptor> tierShuffleDescriptors,
@@ -115,7 +114,6 @@ public class TieredResultPartitionFactory {
numSubpartitions,
bufferSizeBytes,
numAccumulatorExclusiveBuffers,
- memoryDecouplingEnabled,
memoryManager,
isNumberOfPartitionConsumerUndefined);
@@ -166,19 +164,16 @@ public class TieredResultPartitionFactory {
int numSubpartitions,
int bufferSizeBytes,
int numAccumulatorExclusiveBuffers,
- boolean enableMemoryDecoupling,
TieredStorageMemoryManager storageMemoryManager,
boolean isNumberOfPartitionConsumerUndefined) {
- long poolSizeCheckInterval = TieredStorageUtils.getPoolSizeCheckInterval();
BufferAccumulator bufferAccumulator;
- if (enableMemoryDecoupling || (numSubpartitions + 1) > numAccumulatorExclusiveBuffers) {
+ if ((numSubpartitions + 1) > numAccumulatorExclusiveBuffers) {
bufferAccumulator =
new SortBufferAccumulator(
numSubpartitions,
numAccumulatorExclusiveBuffers,
bufferSizeBytes,
- enableMemoryDecoupling ? poolSizeCheckInterval : 0,
storageMemoryManager,
!isNumberOfPartitionConsumerUndefined);
} else {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulator.java
index 8dc6eb2d69f..4c9d3cf7e79 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulator.java
@@ -27,8 +27,6 @@ import org.apache.flink.runtime.io.network.buffer.NetworkBuffer;
import org.apache.flink.runtime.io.network.partition.BufferWithSubpartition;
import org.apache.flink.runtime.io.network.partition.DataBuffer;
import org.apache.flink.runtime.io.network.partition.hybrid.tiered.common.TieredStorageSubpartitionId;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.concurrent.ExecutorThreadFactory;
import org.apache.flink.util.function.TriConsumer;
import javax.annotation.Nullable;
@@ -37,11 +35,6 @@ import javax.annotation.concurrent.GuardedBy;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.LinkedList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
@@ -54,9 +47,9 @@ import static org.apache.flink.util.Preconditions.checkState;
*
* <p>The {@link SortBufferAccumulator} can help use less buffers to accumulate data, which
* decouples the buffer usage with the number of parallelism. The number of buffers used by the
- * {@link SortBufferAccumulator} will be numExpectedBuffers at most. Once the {@link DataBuffer} is
- * full, or switching from broadcast to non-broadcast(or vice versa), the buffer in the sort buffer
- * will be flushed to the tiers.
+ * {@link SortBufferAccumulator} will be numBuffers at most. Once the {@link DataBuffer} is full, or
+ * switching from broadcast to non-broadcast(or vice versa), the buffer in the sort buffer will be
+ * flushed to the tiers.
*
* <p>Note that this class need not be thread-safe, because it should only be accessed from the main
* thread.
@@ -66,8 +59,8 @@ public class SortBufferAccumulator implements BufferAccumulator {
/** The number of the subpartitions. */
private final int numSubpartitions;
- /** The number of the expected buffers used by the {@link SortBufferAccumulator}. */
- private final int numExpectedBuffers;
+ /** The total number of the buffers used by the {@link SortBufferAccumulator}. */
+ private final int numBuffers;
/** The byte size of one single buffer. */
private final int bufferSizeBytes;
@@ -105,18 +98,6 @@ public class SortBufferAccumulator implements BufferAccumulator {
@Nullable
private TriConsumer<TieredStorageSubpartitionId, Buffer, Integer> accumulatedBufferFlusher;
- /**
- * An executor to periodically check the size of buffer pool. If the size is changed, the
- * accumulated buffers should be flushed to release the buffers.
- */
- private final ScheduledExecutorService periodicalAccumulatorFlusher =
- Executors.newSingleThreadScheduledExecutor(
- new ExecutorThreadFactory("hybrid-shuffle-periodical-accumulator-flusher"));
-
- private final long poolSizeCheckInterval;
-
- private AtomicInteger poolSize;
-
/** Whether the current {@link DataBuffer} is a broadcast sort buffer. */
private boolean isBroadcastDataBuffer;
@@ -127,37 +108,20 @@ public class SortBufferAccumulator implements BufferAccumulator {
public SortBufferAccumulator(
int numSubpartitions,
- int numExpectedBuffers,
+ int numBuffers,
int bufferSizeBytes,
- long poolSizeCheckInterval,
TieredStorageMemoryManager memoryManager,
boolean isPartialRecordAllowed) {
this.numSubpartitions = numSubpartitions;
this.bufferSizeBytes = bufferSizeBytes;
- this.numExpectedBuffers = numExpectedBuffers;
- this.poolSizeCheckInterval = poolSizeCheckInterval;
+ this.numBuffers = numBuffers;
this.memoryManager = memoryManager;
this.isPartialRecordAllowed = isPartialRecordAllowed;
- this.poolSize = new AtomicInteger(-1);
}
@Override
public void setup(TriConsumer<TieredStorageSubpartitionId, Buffer, Integer> bufferFlusher) {
this.accumulatedBufferFlusher = bufferFlusher;
-
- if (poolSizeCheckInterval > 0) {
- periodicalAccumulatorFlusher.scheduleWithFixedDelay(
- () -> {
- int newSize = this.memoryManager.getBufferPoolSize();
- int oldSize = poolSize.getAndSet(newSize);
- if (oldSize > newSize) {
- flushCurrentDataBuffer();
- }
- },
- poolSizeCheckInterval,
- poolSizeCheckInterval,
- TimeUnit.MILLISECONDS);
- }
}
@Override
@@ -192,17 +156,6 @@ public class SortBufferAccumulator implements BufferAccumulator {
@Override
public void close() {
- if (periodicalAccumulatorFlusher != null) {
- periodicalAccumulatorFlusher.shutdown();
- try {
- if (!periodicalAccumulatorFlusher.awaitTermination(5L, TimeUnit.MINUTES)) {
- throw new TimeoutException(
- "Timeout for shutting down the periodical accumulator flusher.");
- }
- } catch (Exception e) {
- ExceptionUtils.rethrow(e);
- }
- }
synchronized (lock) {
flushCurrentDataBuffer();
isDataBufferReleased = true;
@@ -248,8 +201,7 @@ public class SortBufferAccumulator implements BufferAccumulator {
@GuardedBy("lock")
private void requestBuffers() {
- while (freeSegments.size()
- < Math.min(numExpectedBuffers, memoryManager.getBufferPoolSize() - 1)) {
+ while (freeSegments.size() < numBuffers) {
Buffer buffer = requestBuffer();
freeSegments.add(checkNotNull(buffer).getMemorySegment());
if (bufferRecycler == null) {
@@ -267,7 +219,7 @@ public class SortBufferAccumulator implements BufferAccumulator {
}
currentDataBuffer.finish();
- while (currentDataBuffer.hasRemaining()) {
+ do {
MemorySegment freeSegment = getFreeSegment();
BufferWithSubpartition bufferWithSubpartition =
currentDataBuffer.getNextBuffer(freeSegment);
@@ -280,7 +232,7 @@ public class SortBufferAccumulator implements BufferAccumulator {
((double) currentDataBuffer.getRecordRemainingBytes())
/ bufferSizeBytes);
flushBuffer(bufferWithSubpartition, numRemainingConsecutiveBuffers);
- }
+ } while (true);
isDataBufferReleased = true;
releaseFreeBuffers();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManager.java
index c2793b72bbb..5bf439064b5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManager.java
@@ -146,9 +146,6 @@ public interface TieredStorageMemoryManager {
*/
void transferBufferOwnership(Object oldOwner, Object newOwner, Buffer buffer);
- /** @return the size of the local buffer pool or -1 if the buffer pool is not initialized. */
- int getBufferPoolSize();
-
/**
* Release all the resources(if exists) and check the state of the {@link
* TieredStorageMemoryManager}.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManagerImpl.java
index 0ef453ec126..7d7aaf43f0d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManagerImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageMemoryManagerImpl.java
@@ -210,16 +210,12 @@ public class TieredStorageMemoryManagerImpl implements TieredStorageMemoryManage
reclaimBuffersIfNeeded(0);
MemorySegment memorySegment = bufferQueue.poll();
- // Requests the buffer as soon as possible from wherever it is available in the buffer pool
- // or the internal queue.
- hardBackpressureTimerGauge.markStart();
- while (memorySegment == null) {
+ if (memorySegment == null) {
memorySegment = requestBufferBlockingFromPool();
- if (memorySegment == null) {
- memorySegment = requestBufferFromQueue();
- }
}
- hardBackpressureTimerGauge.markEnd();
+ if (memorySegment == null) {
+ memorySegment = checkNotNull(requestBufferBlockingFromQueue());
+ }
incNumRequestedBuffer(owner);
return new BufferBuilder(
@@ -261,9 +257,11 @@ public class TieredStorageMemoryManagerImpl implements TieredStorageMemoryManage
while (bufferQueue.size() + numRequestedByGuaranteedReclaimableOwners
< numGuaranteedReclaimableBuffers + numAdditionalBuffers) {
- hardBackpressureTimerGauge.markStart();
+ if (numRequestedBuffers.get() >= bufferPool.getNumBuffers()) {
+ return false;
+ }
+
MemorySegment memorySegment = requestBufferBlockingFromPool();
- hardBackpressureTimerGauge.markEnd();
if (memorySegment == null) {
return false;
}
@@ -290,30 +288,25 @@ public class TieredStorageMemoryManagerImpl implements TieredStorageMemoryManage
try {
releasedStateLock.writeLock().lock();
isReleased = true;
-
- if (executor != null) {
- executor.shutdown();
- try {
- if (!executor.awaitTermination(5L, TimeUnit.MINUTES)) {
- throw new TimeoutException(
- "Timeout for shutting down the buffer reclaim checker executor.");
- }
- } catch (Exception e) {
- ExceptionUtils.rethrow(e);
- }
- }
- while (!bufferQueue.isEmpty()) {
- MemorySegment segment = bufferQueue.poll();
- bufferPool.recycle(segment);
- numRequestedBuffers.decrementAndGet();
- }
} finally {
releasedStateLock.writeLock().unlock();
}
- }
-
- public int getBufferPoolSize() {
- return bufferPool == null ? -1 : bufferPool.getNumBuffers();
+ if (executor != null) {
+ executor.shutdown();
+ try {
+ if (!executor.awaitTermination(5L, TimeUnit.MINUTES)) {
+ throw new TimeoutException(
+ "Timeout for shutting down the buffer reclaim checker executor.");
+ }
+ } catch (Exception e) {
+ ExceptionUtils.rethrow(e);
+ }
+ }
+ while (!bufferQueue.isEmpty()) {
+ MemorySegment segment = bufferQueue.poll();
+ bufferPool.recycle(segment);
+ numRequestedBuffers.decrementAndGet();
+ }
}
/**
@@ -323,6 +316,8 @@ public class TieredStorageMemoryManagerImpl implements TieredStorageMemoryManage
@Nullable
private MemorySegment requestBufferBlockingFromPool() {
MemorySegment memorySegment = null;
+
+ hardBackpressureTimerGauge.markStart();
while (numRequestedBuffers.get() < bufferPool.getNumBuffers()) {
memorySegment = bufferPool.requestMemorySegment();
if (memorySegment == null) {
@@ -339,20 +334,20 @@ public class TieredStorageMemoryManagerImpl implements TieredStorageMemoryManage
break;
}
}
+ hardBackpressureTimerGauge.markEnd();
return memorySegment;
}
/** @return a memory segment from the internal buffer queue. */
- @Nullable
- private MemorySegment requestBufferFromQueue() {
+ private MemorySegment requestBufferBlockingFromQueue() {
CompletableFuture<Void> requestBufferFuture = new CompletableFuture<>();
scheduleCheckRequestBufferFuture(
requestBufferFuture, INITIAL_REQUEST_BUFFER_TIMEOUT_FOR_RECLAIMING_MS);
MemorySegment memorySegment = null;
try {
- memorySegment = bufferQueue.poll(100, TimeUnit.MILLISECONDS);
+ memorySegment = bufferQueue.take();
} catch (InterruptedException e) {
ExceptionUtils.rethrow(e);
} finally {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleUtils.java
index 262f7c476c9..73c54ce1ff4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleUtils.java
@@ -40,6 +40,10 @@ import static org.apache.flink.util.Preconditions.checkState;
*/
public class NettyShuffleUtils {
+ // Temporarily declare the default value here, it would be moved to the configuration class
+ // later.
+ private static final int DEFAULT_HYBRID_SHUFFLE_MIN_BUFFERS = 8;
+
/**
* Calculates and returns the number of required exclusive network buffers per input channel.
*/
@@ -69,32 +73,32 @@ public class NettyShuffleUtils {
final int sortShuffleMinBuffers,
final int numSubpartitions,
final boolean enableTieredStorage,
- final boolean enableMemoryDecoupling,
final int tieredStoreExclusiveBuffers,
- final int tieredStorageMinBuffersPerResultPartition,
final ResultPartitionType type) {
boolean isSortShuffle =
type.isBlockingOrBlockingPersistentResultPartition()
&& numSubpartitions >= sortShuffleMinParallelism;
- boolean isMemoryDecouplingEnabled =
- type.isHybridResultPartition() && enableTieredStorage && enableMemoryDecoupling;
-
- int expected =
- isSortShuffle
- ? sortShuffleMinBuffers
- : (enableTieredStorage
- ? Math.min(tieredStoreExclusiveBuffers, numSubpartitions + 1)
- : (numSubpartitions + 1));
+ int expected;
+ if (isSortShuffle) {
+ expected = sortShuffleMinBuffers;
+ } else {
+ expected =
+ enableTieredStorage
+ ? Math.min(tieredStoreExclusiveBuffers, numSubpartitions + 1)
+ : (numSubpartitions + 1);
+ }
- int min = isMemoryDecouplingEnabled ? tieredStorageMinBuffersPerResultPartition : expected;
- expected = Math.max(min, expected);
+ int min = expected;
+ if (type.isHybridResultPartition()) {
+ min = Math.min(DEFAULT_HYBRID_SHUFFLE_MIN_BUFFERS, expected);
+ }
int max =
type.isBounded()
? numSubpartitions * configuredNetworkBuffersPerChannel
+ numFloatingBuffersPerGate
: (isSortShuffle
- ? Math.max(min, 4 * numSubpartitions)
+ ? 4 * numSubpartitions
: NetworkBufferPool.UNBOUNDED_POOL_SIZE);
max = Math.max(max, expected);
// for each upstream hash-based blocking/pipelined subpartition, at least one buffer is
@@ -166,7 +170,6 @@ public class NettyShuffleUtils {
floatingNetworkBuffersPerGate,
type,
numInputChannels,
- false,
false);
return gateBuffersSpec.getMaxBuffersPerGate();
}
@@ -187,9 +190,7 @@ public class NettyShuffleUtils {
sortShuffleMinBuffers,
numSubpartitions,
false,
- false,
0,
- 1,
type);
// In order to avoid network buffer request timeout (see FLINK-12852), we announce
@@ -198,7 +199,7 @@ public class NettyShuffleUtils {
// back pressure so we need to include all the floating buffers in the announcement, i.e. we
// should take the max value;
// 2. For blocking shuffle, it is back pressure free and floating buffers can be recycled
- // in time, so that the expected buffers would be enough.
+ // in time, so that the minimum required buffers would be enough.
int ret = type.canBePipelinedConsumed() ? tuple.f2 : tuple.f0;
if (ret == Integer.MAX_VALUE) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java
index ae29a8d774f..ac4b15e383c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java
@@ -49,7 +49,6 @@ import java.util.Optional;
import static org.apache.flink.api.common.BatchShuffleMode.ALL_EXCHANGES_HYBRID_FULL;
import static org.apache.flink.api.common.BatchShuffleMode.ALL_EXCHANGES_HYBRID_SELECTIVE;
import static org.apache.flink.configuration.ExecutionOptions.BATCH_SHUFFLE_MODE;
-import static org.apache.flink.configuration.NettyShuffleEnvironmentOptions.NETWORK_HYBRID_SHUFFLE_ENABLE_MEMORY_DECOUPLING;
import static org.apache.flink.configuration.NettyShuffleEnvironmentOptions.NETWORK_HYBRID_SHUFFLE_ENABLE_NEW_MODE;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -118,8 +117,6 @@ public class NettyShuffleEnvironmentConfiguration {
private final long hybridShuffleNumRetainedInMemoryRegionsMax;
- private final boolean isMemoryDecouplingEnabled;
-
private final TieredStorageConfiguration tieredStorageConfiguration;
public NettyShuffleEnvironmentConfiguration(
@@ -148,7 +145,6 @@ public class NettyShuffleEnvironmentConfiguration {
int maxOverdraftBuffersPerGate,
int hybridShuffleSpilledIndexRegionGroupSize,
long hybridShuffleNumRetainedInMemoryRegionsMax,
- boolean isMemoryDecouplingEnabled,
@Nullable TieredStorageConfiguration tieredStorageConfiguration) {
this.numNetworkBuffers = numNetworkBuffers;
@@ -177,7 +173,6 @@ public class NettyShuffleEnvironmentConfiguration {
this.hybridShuffleSpilledIndexRegionGroupSize = hybridShuffleSpilledIndexRegionGroupSize;
this.hybridShuffleNumRetainedInMemoryRegionsMax =
hybridShuffleNumRetainedInMemoryRegionsMax;
- this.isMemoryDecouplingEnabled = isMemoryDecouplingEnabled;
this.tieredStorageConfiguration = tieredStorageConfiguration;
}
@@ -283,10 +278,6 @@ public class NettyShuffleEnvironmentConfiguration {
return hybridShuffleNumRetainedInMemoryRegionsMax;
}
- public boolean isMemoryDecouplingEnabled() {
- return isMemoryDecouplingEnabled;
- }
-
public int getHybridShuffleSpilledIndexRegionGroupSize() {
return hybridShuffleSpilledIndexRegionGroupSize;
}
@@ -419,9 +410,6 @@ public class NettyShuffleEnvironmentConfiguration {
NettyShuffleEnvironmentOptions
.HYBRID_SHUFFLE_NUM_RETAINED_IN_MEMORY_REGIONS_MAX);
- boolean isMemoryDecouplingEnabled =
- configuration.get(NETWORK_HYBRID_SHUFFLE_ENABLE_MEMORY_DECOUPLING);
-
checkArgument(buffersPerChannel >= 0, "Must be non-negative.");
checkArgument(
!maxRequiredBuffersPerGate.isPresent() || maxRequiredBuffersPerGate.get() >= 1,
@@ -468,7 +456,6 @@ public class NettyShuffleEnvironmentConfiguration {
maxOverdraftBuffersPerGate,
hybridShuffleSpilledIndexSegmentSize,
hybridShuffleNumRetainedInMemoryRegionsMax,
- isMemoryDecouplingEnabled,
tieredStorageConfiguration);
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironmentBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironmentBuilder.java
index 00596641441..3ffeafe204b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironmentBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironmentBuilder.java
@@ -102,8 +102,6 @@ public class NettyShuffleEnvironmentBuilder {
private long hybridShuffleNumRetainedInMemoryRegionsMax = Long.MAX_VALUE;
- private boolean isMemoryDecouplingEnabled = false;
-
private int hybridShuffleSpilledIndexSegmentSize = 256;
private TieredStorageConfiguration tieredStorageConfiguration = null;
@@ -242,12 +240,6 @@ public class NettyShuffleEnvironmentBuilder {
return this;
}
- public NettyShuffleEnvironmentBuilder setIsMemoryDecouplingEnabled(
- boolean isMemoryDecouplingEnabled) {
- this.isMemoryDecouplingEnabled = isMemoryDecouplingEnabled;
- return this;
- }
-
public NettyShuffleEnvironmentBuilder setHybridShuffleSpilledIndexSegmentSize(
int hybridShuffleSpilledIndexSegmentSize) {
this.hybridShuffleSpilledIndexSegmentSize = hybridShuffleSpilledIndexSegmentSize;
@@ -288,7 +280,6 @@ public class NettyShuffleEnvironmentBuilder {
maxOverdraftBuffersPerGate,
hybridShuffleSpilledIndexSegmentSize,
hybridShuffleNumRetainedInMemoryRegionsMax,
- isMemoryDecouplingEnabled,
tieredStorageConfiguration),
taskManagerLocation,
new TaskEventDispatcher(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/TestingBufferPool.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/TestingBufferPool.java
index c0c7bbc4e75..e34c4bb8dac 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/TestingBufferPool.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/TestingBufferPool.java
@@ -21,485 +21,118 @@
package org.apache.flink.runtime.io.network.buffer;
import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.core.memory.MemorySegmentFactory;
-import javax.annotation.Nullable;
-
-import java.io.IOException;
import java.util.concurrent.CompletableFuture;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.function.Supplier;
/** Implementation of {@link BufferPool} for testing. */
public class TestingBufferPool implements BufferPool {
- public static final TestingBufferPool NO_OP = TestingBufferPool.builder().build();
-
- private Consumer<Integer> reserveSegmentsConsumer;
-
- private Runnable lazyDestroyRunnable;
-
- private Supplier<Buffer> requestBufferSupplier;
-
- private Supplier<BufferBuilder> requestBufferBuilderSupplier;
-
- private Supplier<BufferBuilder> requestBufferBuilderBlockingSupplier;
-
- private Function<Integer, BufferBuilder> requestBufferBuilderFunction;
-
- private Function<Integer, BufferBuilder> requestBufferBuilderBlockingFunction;
-
- private Function<BufferListener, Boolean> addBufferListenerFunction;
-
- private Supplier<Boolean> isDestroyedSupplier;
-
- private Supplier<MemorySegment> requestMemorySegmentSupplier;
-
- private Supplier<MemorySegment> requestMemorySegmentBlockingSupplier;
-
- private Supplier<Integer> getExpectedNumberOfMemorySegmentsSupplier;
-
- private Supplier<Integer> getMinNumberOfMemorySegmentsSupplier;
-
- private Supplier<Integer> getMaxNumberOfMemorySegmentsSupplier;
-
- private Supplier<Integer> getNumBuffersSupplier;
-
- private Consumer<Integer> setNumBuffersConsumer;
-
- private Consumer<Integer> setMaxOverdraftBuffersPerGateConsumer;
-
- private Supplier<Integer> getMaxOverdraftBuffersPerGateSupplier;
-
- private Supplier<Integer> getNumberOfAvailableMemorySegmentsSupplier;
-
- private Supplier<Integer> bestEffortGetNumOfUsedBuffersSupplier;
-
- private Consumer<MemorySegment> recycleConsumer;
-
- private Supplier<CompletableFuture<?>> getAvailableFutureSupplier;
-
- private TestingBufferPool(
- Consumer<Integer> reserveSegmentsConsumer,
- Runnable lazyDestroyRunnable,
- Supplier<Buffer> requestBufferSupplier,
- Supplier<BufferBuilder> requestBufferBuilderSupplier,
- Supplier<BufferBuilder> requestBufferBuilderBlockingSupplier,
- Function<Integer, BufferBuilder> requestBufferBuilderFunction,
- Function<Integer, BufferBuilder> requestBufferBuilderBlockingFunction,
- Function<BufferListener, Boolean> addBufferListenerFunction,
- Supplier<Boolean> isDestroyedSupplier,
- Supplier<MemorySegment> requestMemorySegmentSupplier,
- Supplier<MemorySegment> requestMemorySegmentBlockingSupplier,
- Supplier<Integer> getExpectedNumberOfMemorySegmentsSupplier,
- Supplier<Integer> getMinNumberOfMemorySegmentsSupplier,
- Supplier<Integer> getMaxNumberOfMemorySegmentsSupplier,
- Supplier<Integer> getNumBuffersSupplier,
- Consumer<Integer> setNumBuffersConsumer,
- Consumer<Integer> setMaxOverdraftBuffersPerGateConsumer,
- Supplier<Integer> getMaxOverdraftBuffersPerGateSupplier,
- Supplier<Integer> getNumberOfAvailableMemorySegmentsSupplier,
- Supplier<Integer> bestEffortGetNumOfUsedBuffersSupplier,
- Consumer<MemorySegment> recycleConsumer,
- Supplier<CompletableFuture<?>> getAvailableFutureSupplier) {
- this.reserveSegmentsConsumer = reserveSegmentsConsumer;
- this.lazyDestroyRunnable = lazyDestroyRunnable;
- this.requestBufferSupplier = requestBufferSupplier;
- this.requestBufferBuilderSupplier = requestBufferBuilderSupplier;
- this.requestBufferBuilderBlockingSupplier = requestBufferBuilderBlockingSupplier;
- this.requestBufferBuilderFunction = requestBufferBuilderFunction;
- this.requestBufferBuilderBlockingFunction = requestBufferBuilderBlockingFunction;
- this.addBufferListenerFunction = addBufferListenerFunction;
- this.isDestroyedSupplier = isDestroyedSupplier;
- this.requestMemorySegmentSupplier = requestMemorySegmentSupplier;
- this.requestMemorySegmentBlockingSupplier = requestMemorySegmentBlockingSupplier;
- this.getExpectedNumberOfMemorySegmentsSupplier = getExpectedNumberOfMemorySegmentsSupplier;
- this.getMinNumberOfMemorySegmentsSupplier = getMinNumberOfMemorySegmentsSupplier;
- this.getMaxNumberOfMemorySegmentsSupplier = getMaxNumberOfMemorySegmentsSupplier;
- this.getNumBuffersSupplier = getNumBuffersSupplier;
- this.setNumBuffersConsumer = setNumBuffersConsumer;
- this.setMaxOverdraftBuffersPerGateConsumer = setMaxOverdraftBuffersPerGateConsumer;
- this.getMaxOverdraftBuffersPerGateSupplier = getMaxOverdraftBuffersPerGateSupplier;
- this.getNumberOfAvailableMemorySegmentsSupplier =
- getNumberOfAvailableMemorySegmentsSupplier;
- this.bestEffortGetNumOfUsedBuffersSupplier = bestEffortGetNumOfUsedBuffersSupplier;
- this.recycleConsumer = recycleConsumer;
- this.getAvailableFutureSupplier = getAvailableFutureSupplier;
- }
-
- public static TestingBufferPool.Builder builder() {
- return new TestingBufferPool.Builder();
- }
-
@Override
- public void reserveSegments(int numberOfSegmentsToReserve) throws IOException {
- reserveSegmentsConsumer.accept(numberOfSegmentsToReserve);
- }
+ public void reserveSegments(int numberOfSegmentsToReserve) {}
@Override
- public void lazyDestroy() {
- lazyDestroyRunnable.run();
- }
+ public void lazyDestroy() {}
- @Nullable
@Override
public Buffer requestBuffer() {
- return requestBufferSupplier.get();
+ MemorySegment segment = requestMemorySegment();
+ return new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE);
}
- @Nullable
@Override
public BufferBuilder requestBufferBuilder() {
- return requestBufferBuilderSupplier.get();
+ throw new UnsupportedOperationException();
}
- @Nullable
@Override
- public BufferBuilder requestBufferBuilder(int targetChannel) {
- return requestBufferBuilderFunction.apply(targetChannel);
+ public BufferBuilder requestBufferBuilderBlocking() throws InterruptedException {
+ throw new UnsupportedOperationException();
}
@Override
- public BufferBuilder requestBufferBuilderBlocking() throws InterruptedException {
- return requestBufferBuilderBlockingSupplier.get();
+ public BufferBuilder requestBufferBuilder(int targetChannel) {
+ throw new UnsupportedOperationException();
}
@Override
public BufferBuilder requestBufferBuilderBlocking(int targetChannel)
throws InterruptedException {
- return requestBufferBuilderBlockingFunction.apply(targetChannel);
+ throw new UnsupportedOperationException();
}
@Override
public boolean addBufferListener(BufferListener listener) {
- return addBufferListenerFunction.apply(listener);
+ throw new UnsupportedOperationException();
}
@Override
public boolean isDestroyed() {
- return isDestroyedSupplier.get();
+ throw new UnsupportedOperationException();
}
- @Nullable
@Override
public MemorySegment requestMemorySegment() {
- return requestMemorySegmentSupplier.get();
+ return MemorySegmentFactory.allocateUnpooledSegment(1024);
}
@Override
public MemorySegment requestMemorySegmentBlocking() throws InterruptedException {
- return requestMemorySegmentBlockingSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int getExpectedNumberOfMemorySegments() {
- return getExpectedNumberOfMemorySegmentsSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int getMinNumberOfMemorySegments() {
- return getMinNumberOfMemorySegmentsSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int getMaxNumberOfMemorySegments() {
- return getMaxNumberOfMemorySegmentsSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int getNumBuffers() {
- return getNumBuffersSupplier.get();
+ return 0;
}
@Override
public void setNumBuffers(int numBuffers) {
- setNumBuffersConsumer.accept(numBuffers);
+ throw new UnsupportedOperationException();
}
@Override
public void setMaxOverdraftBuffersPerGate(int maxOverdraftBuffersPerGate) {
- setMaxOverdraftBuffersPerGateConsumer.accept(maxOverdraftBuffersPerGate);
+ throw new UnsupportedOperationException();
}
@Override
public int getMaxOverdraftBuffersPerGate() {
- return getMaxOverdraftBuffersPerGateSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int getNumberOfAvailableMemorySegments() {
- return getNumberOfAvailableMemorySegmentsSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public int bestEffortGetNumOfUsedBuffers() {
- return bestEffortGetNumOfUsedBuffersSupplier.get();
+ throw new UnsupportedOperationException();
}
@Override
public void recycle(MemorySegment memorySegment) {
- recycleConsumer.accept(memorySegment);
+ throw new UnsupportedOperationException();
}
@Override
public CompletableFuture<?> getAvailableFuture() {
- return getAvailableFutureSupplier.get();
- }
-
- /** Builder for {@link TestingBufferPool}. */
- public static class Builder {
-
- private Consumer<Integer> reserveSegmentsConsumer = (ignore) -> {};
-
- private Runnable lazyDestroyRunnable = () -> {};
-
- private Supplier<Buffer> requestBufferSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<BufferBuilder> requestBufferBuilderSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<BufferBuilder> requestBufferBuilderBlockingSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Function<Integer, BufferBuilder> requestBufferBuilderFunction =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Function<Integer, BufferBuilder> requestBufferBuilderBlockingFunction =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Function<BufferListener, Boolean> addBufferListenerFunction =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Boolean> isDestroyedSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<MemorySegment> requestMemorySegmentSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<MemorySegment> requestMemorySegmentBlockingSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getExpectedNumberOfMemorySegmentsSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getMinNumberOfMemorySegmentsSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getMaxNumberOfMemorySegmentsSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getNumBuffersSupplier = () -> 0;
-
- private Consumer<Integer> setNumBuffersConsumer =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Consumer<Integer> setMaxOverdraftBuffersPerGateConsumer =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getMaxOverdraftBuffersPerGateSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> getNumberOfAvailableMemorySegmentsSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<Integer> bestEffortGetNumOfUsedBuffersSupplier =
- () -> {
- throw new UnsupportedOperationException();
- };
-
- private Consumer<MemorySegment> recycleConsumer =
- (ignore) -> {
- throw new UnsupportedOperationException();
- };
-
- private Supplier<CompletableFuture<?>> getAvailableFutureSupplier = () -> AVAILABLE;
-
- private Builder() {}
-
- public TestingBufferPool build() {
- return new TestingBufferPool(
- reserveSegmentsConsumer,
- lazyDestroyRunnable,
- requestBufferSupplier,
- requestBufferBuilderSupplier,
- requestBufferBuilderBlockingSupplier,
- requestBufferBuilderFunction,
- requestBufferBuilderBlockingFunction,
- addBufferListenerFunction,
- isDestroyedSupplier,
- requestMemorySegmentSupplier,
- requestMemorySegmentBlockingSupplier,
- getExpectedNumberOfMemorySegmentsSupplier,
- getMinNumberOfMemorySegmentsSupplier,
- getMaxNumberOfMemorySegmentsSupplier,
- getNumBuffersSupplier,
- setNumBuffersConsumer,
- setMaxOverdraftBuffersPerGateConsumer,
- getMaxOverdraftBuffersPerGateSupplier,
- getNumberOfAvailableMemorySegmentsSupplier,
- bestEffortGetNumOfUsedBuffersSupplier,
- recycleConsumer,
- getAvailableFutureSupplier);
- }
-
- public Builder setReserveSegmentsConsumer(Consumer<Integer> reserveSegmentsConsumer) {
- this.reserveSegmentsConsumer = reserveSegmentsConsumer;
- return this;
- }
-
- public Builder setLazyDestroyRunnable(Runnable lazyDestroyRunnable) {
- this.lazyDestroyRunnable = lazyDestroyRunnable;
- return this;
- }
-
- public Builder setRequestBufferSupplier(Supplier<Buffer> requestBufferSupplier) {
- this.requestBufferSupplier = requestBufferSupplier;
- return this;
- }
-
- public Builder setRequestBufferBuilderSupplier(
- Supplier<BufferBuilder> requestBufferBuilderSupplier) {
- this.requestBufferBuilderSupplier = requestBufferBuilderSupplier;
- return this;
- }
-
- public Builder setRequestBufferBuilderBlockingSupplier(
- Supplier<BufferBuilder> requestBufferBuilderBlockingSupplier) {
- this.requestBufferBuilderBlockingSupplier = requestBufferBuilderBlockingSupplier;
- return this;
- }
-
- public Builder setRequestBufferBuilderFunction(
- Function<Integer, BufferBuilder> requestBufferBuilderFunction) {
- this.requestBufferBuilderFunction = requestBufferBuilderFunction;
- return this;
- }
-
- public Builder setRequestBufferBuilderBlockingFunction(
- Function<Integer, BufferBuilder> requestBufferBuilderBlockingFunction) {
- this.requestBufferBuilderBlockingFunction = requestBufferBuilderBlockingFunction;
- return this;
- }
-
- public Builder setAddBufferListenerFunction(
- Function<BufferListener, Boolean> addBufferListenerFunction) {
- this.addBufferListenerFunction = addBufferListenerFunction;
- return this;
- }
-
- public Builder setIsDestroyedSupplier(Supplier<Boolean> isDestroyedSupplier) {
- this.isDestroyedSupplier = isDestroyedSupplier;
- return this;
- }
-
- public Builder setRequestMemorySegmentSupplier(
- Supplier<MemorySegment> requestMemorySegmentSupplier) {
- this.requestMemorySegmentSupplier = requestMemorySegmentSupplier;
- return this;
- }
-
- public Builder setRequestMemorySegmentBlockingSupplier(
- Supplier<MemorySegment> requestMemorySegmentBlockingSupplier) {
- this.requestMemorySegmentBlockingSupplier = requestMemorySegmentBlockingSupplier;
- return this;
- }
-
- public Builder setGetExpectedNumberOfMemorySegmentsSupplier(
- Supplier<Integer> getExpectedNumberOfMemorySegmentsSupplier) {
- this.getExpectedNumberOfMemorySegmentsSupplier =
- getExpectedNumberOfMemorySegmentsSupplier;
- return this;
- }
-
- public Builder setGetMinNumberOfMemorySegmentsSupplier(
- Supplier<Integer> getMinNumberOfMemorySegmentsSupplier) {
- this.getMinNumberOfMemorySegmentsSupplier = getMinNumberOfMemorySegmentsSupplier;
- return this;
- }
-
- public Builder setGetMaxNumberOfMemorySegmentsSupplier(
- Supplier<Integer> getMaxNumberOfMemorySegmentsSupplier) {
- this.getMaxNumberOfMemorySegmentsSupplier = getMaxNumberOfMemorySegmentsSupplier;
- return this;
- }
-
- public Builder setGetNumBuffersSupplier(Supplier<Integer> getNumBuffersSupplier) {
- this.getNumBuffersSupplier = getNumBuffersSupplier;
- return this;
- }
-
- public Builder setSetNumBuffersConsumer(Consumer<Integer> setNumBuffersConsumer) {
- this.setNumBuffersConsumer = setNumBuffersConsumer;
- return this;
- }
-
- public Builder setSetMaxOverdraftBuffersPerGateConsumer(
- Consumer<Integer> setMaxOverdraftBuffersPerGateConsumer) {
- this.setMaxOverdraftBuffersPerGateConsumer = setMaxOverdraftBuffersPerGateConsumer;
- return this;
- }
-
- public Builder setGetMaxOverdraftBuffersPerGateSupplier(
- Supplier<Integer> getMaxOverdraftBuffersPerGateSupplier) {
- this.getMaxOverdraftBuffersPerGateSupplier = getMaxOverdraftBuffersPerGateSupplier;
- return this;
- }
-
- public Builder setGetNumberOfAvailableMemorySegmentsSupplier(
- Supplier<Integer> getNumberOfAvailableMemorySegmentsSupplier) {
- this.getNumberOfAvailableMemorySegmentsSupplier =
- getNumberOfAvailableMemorySegmentsSupplier;
- return this;
- }
-
- public Builder setBestEffortGetNumOfUsedBuffersSupplier(
- Supplier<Integer> bestEffortGetNumOfUsedBuffersSupplier) {
- this.bestEffortGetNumOfUsedBuffersSupplier = bestEffortGetNumOfUsedBuffersSupplier;
- return this;
- }
-
- public Builder setRecycleConsumer(Consumer<MemorySegment> recycleConsumer) {
- this.recycleConsumer = recycleConsumer;
- return this;
- }
-
- public Builder setGetAvailableFutureSupplier(
- Supplier<CompletableFuture<?>> getAvailableFutureSupplier) {
- this.getAvailableFutureSupplier = getAvailableFutureSupplier;
- return this;
- }
+ return AVAILABLE;
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
index 8a6c440def3..7d78fba72bb 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
@@ -19,7 +19,6 @@
package org.apache.flink.runtime.io.network.netty;
import org.apache.flink.configuration.NettyShuffleEnvironmentOptions.CompressionCodec;
-import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.metrics.SimpleCounter;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -596,11 +595,7 @@ class CreditBasedPartitionRequestClientHandlerTest {
final String expectedMessage = "test exception on buffer";
final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
- inputGate.setBufferPool(
- TestingBufferPool.builder()
- .setRequestMemorySegmentSupplier(
- () -> MemorySegmentFactory.allocateUnpooledSegment(1024))
- .build());
+ inputGate.setBufferPool(new TestingBufferPool());
final RemoteInputChannel inputChannel =
new TestRemoteInputChannelForError(inputGate, expectedMessage);
final CreditBasedPartitionRequestClientHandler handler =
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientDecoderDelegateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientDecoderDelegateTest.java
index 90c46267800..96611550686 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientDecoderDelegateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientDecoderDelegateTest.java
@@ -78,11 +78,7 @@ class NettyMessageClientDecoderDelegateTest {
channel = new EmbeddedChannel(new NettyMessageClientDecoderDelegate(handler));
inputGate = createSingleInputGate(1, networkBufferPool);
- inputGate.setBufferPool(
- TestingBufferPool.builder()
- .setRequestMemorySegmentSupplier(
- () -> MemorySegmentFactory.allocateUnpooledSegment(1024))
- .build());
+ inputGate.setBufferPool(new TestingBufferPool());
RemoteInputChannel inputChannel =
createRemoteInputChannel(
inputGate, new TestingPartitionRequestClient(), NUMBER_OF_BUFFER_RESPONSES);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientSideSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientSideSerializationTest.java
index 39549a5623a..8f4123a08a6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientSideSerializationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageClientSideSerializationTest.java
@@ -81,11 +81,7 @@ class NettyMessageClientSideSerializationTest {
void setup() throws IOException, InterruptedException {
networkBufferPool = new NetworkBufferPool(8, BUFFER_SIZE);
inputGate = createSingleInputGate(1, networkBufferPool);
- inputGate.setBufferPool(
- TestingBufferPool.builder()
- .setRequestMemorySegmentSupplier(
- () -> MemorySegmentFactory.allocateUnpooledSegment(1024))
- .build());
+ inputGate.setBufferPool(new TestingBufferPool());
RemoteInputChannel inputChannel =
createRemoteInputChannel(inputGate, new TestingPartitionRequestClient());
inputChannel.requestSubpartitions();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java
index 8be355781f9..419bf508d8e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition;
-import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.runtime.io.network.ConnectionManager;
import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
@@ -273,11 +272,7 @@ public class InputGateFairnessTest {
channels[11].onBuffer(mockBuffer, 0, -1, 0);
channelSequenceNums[11]++;
- gate.setBufferPool(
- TestingBufferPool.builder()
- .setRequestMemorySegmentSupplier(
- () -> MemorySegmentFactory.allocateUnpooledSegment(1024))
- .build());
+ gate.setBufferPool(new TestingBufferPool());
gate.setInputChannels(channels);
gate.setupChannels();
gate.requestPartitions();
@@ -358,7 +353,7 @@ public class InputGateFairnessTest {
private static class FairnessVerifyingInputGate extends SingleInputGate {
private static final int BUFFER_SIZE = 32 * 1024;
private static final SupplierWithException<BufferPool, IOException>
- STUB_BUFFER_POOL_FACTORY = () -> TestingBufferPool.NO_OP;
+ STUB_BUFFER_POOL_FACTORY = TestingBufferPool::new;
private final PrioritizedDeque<InputChannel> channelsWithData;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionBuilder.java
index 88e5f64dcdf..b53a23ca93f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionBuilder.java
@@ -94,8 +94,6 @@ public class ResultPartitionBuilder {
private long hybridShuffleNumRetainedInMemoryRegionsMax = Long.MAX_VALUE;
- private boolean isMemoryDecouplingEnabled = false;
-
public ResultPartitionBuilder setResultPartitionIndex(int partitionIndex) {
this.partitionIndex = partitionIndex;
return this;
@@ -236,11 +234,6 @@ public class ResultPartitionBuilder {
return this;
}
- public ResultPartitionBuilder setIsMemoryDecouplingEnabled(boolean isMemoryDecouplingEnabled) {
- this.isMemoryDecouplingEnabled = isMemoryDecouplingEnabled;
- return this;
- }
-
public ResultPartitionBuilder setHybridShuffleSpilledIndexRegionGroupSize(
int hybridShuffleSpilledIndexRegionGroupSize) {
this.hybridShuffleSpilledIndexRegionGroupSize = hybridShuffleSpilledIndexRegionGroupSize;
@@ -268,7 +261,6 @@ public class ResultPartitionBuilder {
maxOverdraftBuffersPerGate,
hybridShuffleSpilledIndexRegionGroupSize,
hybridShuffleNumRetainedInMemoryRegionsMax,
- isMemoryDecouplingEnabled,
Optional.empty());
SupplierWithException<BufferPool, IOException> factory =
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
index 88abb961672..0ba2e01908d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
@@ -221,7 +221,6 @@ class ResultPartitionFactoryTest {
0,
256,
Long.MAX_VALUE,
- false,
tieredStorage);
final ResultPartitionDeploymentDescriptor descriptor =
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/GateBuffersSpecTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/GateBuffersSpecTest.java
index fbab00c57a2..1652f80fe91 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/GateBuffersSpecTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/GateBuffersSpecTest.java
@@ -121,9 +121,9 @@ class GateBuffersSpecTest {
createGateBuffersSpec(
numInputChannels, partitionType, numExclusiveBuffersPerChannel);
- int numExclusivePerChannel = 0;
int minBuffersPerGate = 1;
int maxBuffersPerGate = 8;
+ int numExclusivePerChannel = 0;
checkBuffersInGate(
gateBuffersSpec,
@@ -173,36 +173,6 @@ class GateBuffersSpecTest {
assertThat(effectiveMaxRequiredBuffers).isEqualTo(expectEffectiveMaxRequiredBuffers);
}
- @ParameterizedTest
- @MethodSource("parameters")
- void testDefaultMinBuffersPerGate(ResultPartitionType partitionType) {
- int numInputChannels = 499;
- int numExclusiveBuffers = 2;
-
- GateBuffersSpec gateBuffersSpec =
- createGateBuffersSpec(
- numInputChannels, partitionType, numExclusiveBuffers, false, false);
- int expectedBuffersPerGate = 999;
- int maxBuffersPerGate = 1006;
- checkBuffersInGate(
- gateBuffersSpec,
- numExclusiveBuffers,
- expectedBuffersPerGate,
- expectedBuffersPerGate,
- maxBuffersPerGate);
-
- gateBuffersSpec =
- createGateBuffersSpec(
- numInputChannels, partitionType, numExclusiveBuffers, true, true);
- int minBuffersPerGate = partitionType.isHybridResultPartition() ? 2 : 999;
- checkBuffersInGate(
- gateBuffersSpec,
- numExclusiveBuffers,
- expectedBuffersPerGate,
- minBuffersPerGate,
- maxBuffersPerGate);
- }
-
private static void checkBuffersInGate(
GateBuffersSpec gateBuffersSpec,
int numExclusivePerChannel,
@@ -231,26 +201,9 @@ class GateBuffersSpecTest {
8,
partitionType,
numInputChannels,
- false,
false);
}
- private static GateBuffersSpec createGateBuffersSpec(
- int numInputChannels,
- ResultPartitionType partitionType,
- int numExclusiveBuffersPerChannel,
- boolean enableTieredStorage,
- boolean memoryDecouplingEnabled) {
- return InputGateSpecUtils.createGateBuffersSpec(
- getMaxRequiredBuffersPerGate(partitionType),
- numExclusiveBuffersPerChannel,
- 8,
- partitionType,
- numInputChannels,
- enableTieredStorage,
- memoryDecouplingEnabled);
- }
-
private static Optional<Integer> getMaxRequiredBuffersPerGate(
ResultPartitionType partitionType) {
return isPipelinedOrHybridResultPartition(partitionType)
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
index 0dc60e9e3d6..6abadbf10ea 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition.consumer;
-import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
@@ -40,8 +39,6 @@ import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.buffer.Buffer.DataType;
import org.apache.flink.runtime.io.network.buffer.BufferBuilder;
import org.apache.flink.runtime.io.network.buffer.BufferPool;
-import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler;
-import org.apache.flink.runtime.io.network.buffer.NetworkBuffer;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.io.network.buffer.TestingBufferPool;
import org.apache.flink.runtime.io.network.partition.InputChannelTestUtils;
@@ -1378,14 +1375,7 @@ class RemoteInputChannelTest {
@Test
void testOnUpstreamBlockedAndResumed() throws Exception {
- BufferPool bufferPool =
- TestingBufferPool.builder()
- .setRequestBufferSupplier(
- () ->
- new NetworkBuffer(
- MemorySegmentFactory.allocateUnpooledSegment(1024),
- FreeingBufferRecycler.INSTANCE))
- .build();
+ BufferPool bufferPool = new TestingBufferPool();
SingleInputGate inputGate = createSingleInputGate(bufferPool);
RemoteInputChannel remoteChannel1 = createRemoteInputChannel(inputGate, 0, 2);
@@ -1434,14 +1424,7 @@ class RemoteInputChannelTest {
@Test
void testRequestBuffer() throws Exception {
- BufferPool bufferPool =
- TestingBufferPool.builder()
- .setRequestBufferSupplier(
- () ->
- new NetworkBuffer(
- MemorySegmentFactory.allocateUnpooledSegment(1024),
- FreeingBufferRecycler.INSTANCE))
- .build();
+ BufferPool bufferPool = new TestingBufferPool();
SingleInputGate inputGate = createSingleInputGate(bufferPool);
RemoteInputChannel remoteChannel1 = createRemoteInputChannel(inputGate, 0, 2);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateBuilder.java
index 99226e0c1d0..ab89c226c29 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateBuilder.java
@@ -70,13 +70,15 @@ public class SingleInputGateBuilder {
private MemorySegmentProvider segmentProvider =
InputChannelTestUtils.StubMemorySegmentProvider.getInstance();
+ private BufferPool bufferPool = null;
+
private ChannelStateWriter channelStateWriter = ChannelStateWriter.NO_OP;
@Nullable
private BiFunction<InputChannelBuilder, SingleInputGate, InputChannel> channelFactory = null;
private SupplierWithException<BufferPool, IOException> bufferPoolFactory =
- () -> TestingBufferPool.NO_OP;
+ TestingBufferPool::new;
private BufferDebloatConfiguration bufferDebloatConfiguration =
BufferDebloatConfiguration.fromConfiguration(new Configuration());
private Function<BufferDebloatConfiguration, ThroughputCalculator> createThroughputCalculator =
@@ -147,6 +149,11 @@ public class SingleInputGateBuilder {
return this;
}
+ public SingleInputGateBuilder setBufferPool(BufferPool bufferPool) {
+ this.bufferPool = bufferPool;
+ return this;
+ }
+
/** Adds automatic initialization of all channels with the given factory. */
public SingleInputGateBuilder setChannelFactory(
BiFunction<InputChannelBuilder, SingleInputGate, InputChannel> channelFactory) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/TestingTieredStorageMemoryManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/TestingTieredStorageMemoryManager.java
index 649346ca8e5..a4b2349fd34 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/TestingTieredStorageMemoryManager.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/TestingTieredStorageMemoryManager.java
@@ -53,8 +53,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
private final TriConsumer<Object, Object, Buffer> transferBufferOwnershipConsumer;
- private final Supplier<Integer> getBufferPoolSizeSupplier;
-
private final Runnable releaseRunnable;
private TestingTieredStorageMemoryManager(
@@ -67,7 +65,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
Function<Integer, Boolean> ensureCapacityFunction,
Function<Object, Integer> numOwnerRequestedBufferFunction,
TriConsumer<Object, Object, Buffer> transferBufferOwnershipConsumer,
- Supplier<Integer> getBufferPoolSizeSupplier,
Runnable releaseRunnable) {
this.setupConsumer = setupConsumer;
this.setMetricGroupConsumer = setMetricGroupConsumer;
@@ -78,7 +75,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
this.ensureCapacityFunction = ensureCapacityFunction;
this.numOwnerRequestedBufferFunction = numOwnerRequestedBufferFunction;
this.transferBufferOwnershipConsumer = transferBufferOwnershipConsumer;
- this.getBufferPoolSizeSupplier = getBufferPoolSizeSupplier;
this.releaseRunnable = releaseRunnable;
}
@@ -127,11 +123,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
transferBufferOwnershipConsumer.accept(oldOwner, newOwner, buffer);
}
- @Override
- public int getBufferPoolSize() {
- return getBufferPoolSizeSupplier.get();
- }
-
@Override
public void release() {
releaseRunnable.run();
@@ -149,8 +140,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
private Consumer<Runnable> listenBufferReclaimRequestConsumer = runnable -> {};
- private Function<Object, BufferBuilder> requestBufferFunction = owner -> null;
-
private Function<Object, BufferBuilder> requestBufferBlockingFunction = owner -> null;
private Function<Object, Integer> getMaxNonReclaimableBuffersFunction = owner -> 0;
@@ -162,8 +151,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
private TriConsumer<Object, Object, Buffer> transferBufferOwnershipConsumer =
(oldOwner, newOwner, buffer) -> {};
- private Supplier<Integer> getBufferPoolSizeSupplier = () -> 0;
-
private Runnable releaseRunnable = () -> {};
public Builder() {}
@@ -216,12 +203,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
return this;
}
- public TestingTieredStorageMemoryManager.Builder setGetBufferPoolSizeSupplier(
- Supplier<Integer> getBufferPoolSizeSupplier) {
- this.getBufferPoolSizeSupplier = getBufferPoolSizeSupplier;
- return this;
- }
-
public TestingTieredStorageMemoryManager.Builder setReleaseRunnable(
Runnable releaseRunnable) {
this.releaseRunnable = releaseRunnable;
@@ -239,7 +220,6 @@ public class TestingTieredStorageMemoryManager implements TieredStorageMemoryMan
ensureCapacityFunction,
numOwnerRequestedBufferFunction,
transferBufferOwnershipConsumer,
- getBufferPoolSizeSupplier,
releaseRunnable);
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionTest.java
index e2d771a5c72..865464fc1e0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TieredResultPartitionTest.java
@@ -147,9 +147,8 @@ class TieredResultPartitionTest {
@Test
void testMinMaxNetworkBuffersTieredResultPartition() {
- final int numSubpartitions = 105;
- final int tieredStorageTotalExclusiveBufferNum = 103;
- final int tieredStorageMaxBuffersPerRecord = 8;
+ int numSubpartitions = 105;
+ int tieredStorageTotalExclusiveBufferNum = 103;
Tuple3<Integer, Integer, Integer> minMaxNetworkBuffers =
getMinMaxNetworkBuffersPerResultPartition(
100,
@@ -158,12 +157,10 @@ class TieredResultPartitionTest {
10,
numSubpartitions,
true,
- true,
tieredStorageTotalExclusiveBufferNum,
- tieredStorageMaxBuffersPerRecord,
ResultPartitionType.HYBRID_SELECTIVE);
assertThat(minMaxNetworkBuffers.f0).isEqualTo(tieredStorageTotalExclusiveBufferNum);
- assertThat(minMaxNetworkBuffers.f1).isEqualTo(tieredStorageMaxBuffersPerRecord);
+ assertThat(minMaxNetworkBuffers.f1).isEqualTo(8);
assertThat(minMaxNetworkBuffers.f2).isEqualTo(Integer.MAX_VALUE);
}
@@ -296,7 +293,6 @@ class TieredResultPartitionTest {
Integer.MAX_VALUE,
NETWORK_BUFFER_SIZE,
isBroadcastOnly,
- true,
new ResultPartitionManager(),
new BufferCompressor(NETWORK_BUFFER_SIZE, CompressionCodec.LZ4),
tierShuffleDescriptors,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulatorTest.java
index 62ebaeb5f98..798ee7d7bc4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/SortBufferAccumulatorTest.java
@@ -95,7 +95,7 @@ class SortBufferAccumulatorTest {
// flushed.
try (SortBufferAccumulator bufferAccumulator =
new SortBufferAccumulator(
- 1, 2, BUFFER_SIZE_BYTES, 0L, memoryManager, isPartialRecordAllowed)) {
+ 1, 2, BUFFER_SIZE_BYTES, memoryManager, isPartialRecordAllowed)) {
bufferAccumulator.setup(
((subpartition, buffer, numRemainingBuffers) -> {
assertThat(buffer.getDataType()).isIn(expectedDataTypes);
@@ -144,7 +144,7 @@ class SortBufferAccumulatorTest {
// flushed.
try (SortBufferAccumulator bufferAccumulator =
new SortBufferAccumulator(
- 1, 2, BUFFER_SIZE_BYTES, 0L, memoryManager, isPartialRecordAllowed)) {
+ 1, 2, BUFFER_SIZE_BYTES, memoryManager, isPartialRecordAllowed)) {
AtomicInteger numReceivedBuffers = new AtomicInteger(0);
bufferAccumulator.setup(
(subpartitionIndex, buffer, numRemainingBuffers) -> {
@@ -170,8 +170,8 @@ class SortBufferAccumulatorTest {
TieredStorageMemoryManager memoryManager = createStorageMemoryManager(numBuffers);
try (SortBufferAccumulator bufferAccumulator =
- new SortBufferAccumulator(1, 1, bufferSize, 0L, memoryManager, true)) {
- bufferAccumulator.setup((subpartitionIndex, buffers, numRemainingBuffers) -> {});
+ new SortBufferAccumulator(1, 1, bufferSize, memoryManager, true)) {
+ bufferAccumulator.setup((subpartitionIndex, buffer, numRemainingBuffers) -> {});
assertThatThrownBy(
() ->
bufferAccumulator.receive(
@@ -191,7 +191,7 @@ class SortBufferAccumulatorTest {
createStorageMemoryManager(numBuffers);
SortBufferAccumulator bufferAccumulator =
new SortBufferAccumulator(
- 1, 2, BUFFER_SIZE_BYTES, 0L, tieredStorageMemoryManager, true);
+ 1, 2, BUFFER_SIZE_BYTES, tieredStorageMemoryManager, true);
bufferAccumulator.setup(
((subpartition, buffer, numRemainingBuffers) -> buffer.recycleBuffer()));
bufferAccumulator.receive(
@@ -221,17 +221,16 @@ class SortBufferAccumulatorTest {
// When flushing data from the data buffer, the buffers should
// not be requested from here anymore because the freeSegments
// can be reused before released.
- assertThat(numRequestedBuffers.get() < numBuffers).isTrue();
+ assertThat(numRequestedBuffers.get() <= numBuffers).isTrue();
return new BufferBuilder(
memorySegment,
segment -> globalPool.requestPooledMemorySegment());
})
- .setGetBufferPoolSizeSupplier(() -> numBuffers)
.build();
SortBufferAccumulator bufferAccumulator =
new SortBufferAccumulator(
- numSubpartitions, numBuffers, BUFFER_SIZE_BYTES, 0L, memoryManager, true);
+ numSubpartitions, numBuffers, BUFFER_SIZE_BYTES, memoryManager, true);
bufferAccumulator.setup(
((subpartition, buffer, numRemainingBuffers) -> buffer.recycleBuffer()));
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/BatchShuffleITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/BatchShuffleITCaseBase.java
index 6c693d5e373..3f03c06d287 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/BatchShuffleITCaseBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/BatchShuffleITCaseBase.java
@@ -136,16 +136,15 @@ class BatchShuffleITCaseBase {
throws Exception {
JobGraphRunningUtil.execute(
jobGraph, configuration, NUM_TASK_MANAGERS, NUM_SLOTS_PER_TASK_MANAGER);
- checkAllDataReceived(numRecordsToSend, PARALLELISM, 1);
+ checkAllDataReceived(numRecordsToSend);
}
- protected void checkAllDataReceived(
- int numRecordsToSend, int parallelism, int numResultPartitions) {
+ private void checkAllDataReceived(int numRecordsToSend) {
assertThat(Arrays.stream(NUM_RECEIVED_RECORDS).sum())
- .isEqualTo(numRecordsToSend * parallelism * numResultPartitions);
+ .isEqualTo(numRecordsToSend * PARALLELISM);
}
- static class StringSource implements ParallelSourceFunction<String> {
+ private static class StringSource implements ParallelSourceFunction<String> {
private volatile boolean isRunning = true;
private int numRecordsToSend;
@@ -166,7 +165,7 @@ class BatchShuffleITCaseBase {
}
}
- static class VerifySink extends RichSinkFunction<String> {
+ private static class VerifySink extends RichSinkFunction<String> {
private final boolean failExecution;
private final boolean deletePartitionFile;
@@ -181,14 +180,9 @@ class BatchShuffleITCaseBase {
@Override
public void open(OpenContext openContext) throws Exception {
- int indexOfSubtask = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
- int attemptNumber = getRuntimeContext().getTaskInfo().getAttemptNumber();
-
- if (attemptNumber > 0) {
- NUM_RECEIVED_RECORDS[indexOfSubtask] = 0;
- }
-
- if (attemptNumber > 0 || indexOfSubtask != 0) {
+ NUM_RECEIVED_RECORDS[getRuntimeContext().getTaskInfo().getIndexOfThisSubtask()] = 0;
+ if (getRuntimeContext().getTaskInfo().getAttemptNumber() > 0
+ || getRuntimeContext().getTaskInfo().getIndexOfThisSubtask() != 0) {
return;
}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/HybridShuffleMemoryDecouplingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/HybridShuffleMemoryDecouplingITCase.java
deleted file mode 100644
index 89b00590334..00000000000
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/HybridShuffleMemoryDecouplingITCase.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.runtime;
-
-import org.apache.flink.api.common.BatchShuffleMode;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.connector.source.Boundedness;
-import org.apache.flink.configuration.BatchExecutionOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.ExecutionOptions;
-import org.apache.flink.configuration.NettyShuffleEnvironmentOptions;
-import org.apache.flink.configuration.TaskManagerOptions;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.JobType;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.graph.StreamGraph;
-import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator;
-import org.apache.flink.streaming.api.operators.StreamSource;
-
-import org.junit.jupiter.api.Test;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests for hybrid shuffle with memory decoupling enabled. */
-public class HybridShuffleMemoryDecouplingITCase extends BatchShuffleITCaseBase {
-
- private static final int NUM_RECORDS = 10000;
- private static final int NUM_RESULT_PARTITIONS = 10;
- private static final int PARALLELISM = 10;
-
- @Test
- void testHybridWithMultipleResultPartitionsCauseInsufficientBuffers() {
- Configuration configuration = configureHybridOptions(getConfiguration(), false);
- JobGraph jobGraph =
- createJobGraphWithMultipleResultPartitions(
- NUM_RECORDS, false, NUM_RESULT_PARTITIONS, PARALLELISM, configuration);
-
- assertThatThrownBy(() -> JobGraphRunningUtil.execute(jobGraph, configuration, 1, 1))
- .hasStackTraceContaining("Insufficient number of network buffers");
- }
-
- @Test
- void testHybridWithMultipleResultPartitionsWithMemoryDecoupling() throws Exception {
- Configuration configuration = configureHybridOptions(getConfiguration(), true);
- JobGraph jobGraph =
- createJobGraphWithMultipleResultPartitions(
- NUM_RECORDS, false, NUM_RESULT_PARTITIONS, PARALLELISM, configuration);
-
- JobGraphRunningUtil.execute(jobGraph, configuration, 1, 1);
- checkAllDataReceived(NUM_RECORDS, PARALLELISM, NUM_RESULT_PARTITIONS);
- }
-
- private JobGraph createJobGraphWithMultipleResultPartitions(
- int numRecordsToSend,
- boolean failExecution,
- int numResultPartitions,
- int parallelism,
- Configuration configuration) {
- configuration.setBoolean(BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_ENABLED, false);
- StreamExecutionEnvironment env =
- StreamExecutionEnvironment.getExecutionEnvironment(configuration);
- env.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, 0L));
- env.setParallelism(parallelism);
-
- DataStream<String> source =
- new DataStreamSource<>(
- env,
- BasicTypeInfo.STRING_TYPE_INFO,
- new StreamSource<>(new StringSource(numRecordsToSend)),
- true,
- "source",
- Boundedness.BOUNDED);
-
- for (int i = 0; i < numResultPartitions; i++) {
- source.map(value -> value).rebalance().addSink(new VerifySink(failExecution, false));
- }
-
- StreamGraph streamGraph = env.getStreamGraph();
- streamGraph.setJobType(JobType.BATCH);
- return StreamingJobGraphGenerator.createJobGraph(streamGraph);
- }
-
- private Configuration configureHybridOptions(
- Configuration configuration, boolean enableMemoryDecoupling) {
- configuration.set(
- ExecutionOptions.BATCH_SHUFFLE_MODE,
- BatchShuffleMode.ALL_EXCHANGES_HYBRID_SELECTIVE);
- configuration.setBoolean(
- NettyShuffleEnvironmentOptions.NETWORK_HYBRID_SHUFFLE_ENABLE_MEMORY_DECOUPLING,
- enableMemoryDecoupling);
- configuration.setString(TaskManagerOptions.NETWORK_MEMORY_MAX.key(), "3200k");
-
- return configuration;
- }
-}