You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2021/10/12 13:54:04 UTC
[flink] 02/03: [refactor][streaming] Builder for parameters of
BufferDebloaterTest
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 0300df70d95cf62c33781f1f0f8de74a54ed030a
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Mon Oct 11 18:35:23 2021 +0200
[refactor][streaming] Builder for parameters of BufferDebloaterTest
---
.../tasks/bufferdebloat/BufferDebloaterTest.java | 179 +++++++++++++++------
1 file changed, 131 insertions(+), 48 deletions(-)
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
index 4dca4c2..c76e4ef 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
@@ -45,29 +45,54 @@ public class BufferDebloaterTest extends TestLogger {
@Test
public void testZeroBuffersInUse() {
// if the gate returns the zero buffers in use it should be transformed to 1.
- testBufferSizeCalculation(3, asList(0, 1, 0), 3333, 50, 2400, 1000, 1111);
+ testBufferDebloater()
+ .withDebloatTarget(1000)
+ .withBufferSize(50, 2400)
+ .withNumberOfBuffersInUse(asList(0, 1, 0))
+ .withThroughput(3333)
+ .expectBufferSize(1111);
}
@Test
public void testCorrectBufferSizeCalculation() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, 1200, 249);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, 1100)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(249);
}
@Test
public void testCalculatedBufferSizeLessThanMin() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 250, 1100, 1200, 250);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(250, 1100)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(250);
}
@Test
public void testCalculatedBufferSizeForThroughputZero() {
// When the throughput is zero then min buffer size will be taken.
- testBufferSizeCalculation(3, asList(3, 5, 8), 0, 50, 1100, 1200, 50);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, 1100)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(0)
+ .expectBufferSize(50);
}
@Test
public void testConfiguredConsumptionTimeIsTooLow() {
// When the consumption time is low then min buffer size will be taken.
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, 7, 50);
+ testBufferDebloater()
+ .withDebloatTarget(7)
+ .withBufferSize(50, 1100)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(50);
}
@Test
@@ -75,7 +100,12 @@ public class BufferDebloaterTest extends TestLogger {
// New calculated buffer size should be more than max value it means that we should take max
// value which means that no updates should happen(-1 means that we take the initial value)
// because the old value equal to new value.
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 248, 1200, -1);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, 248)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(-1);
}
@Test
@@ -83,64 +113,52 @@ public class BufferDebloaterTest extends TestLogger {
// New calculated buffer size should be a little less than current value(or max value which
// is the same) it means that no updates should happen(-1 means that we take the initial
// value) because the new value is not so different from the old one.
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 250, 1200, -1);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, 250)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(-1);
}
@Test(expected = IllegalArgumentException.class)
public void testNegativeMinBufferSize() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, -1, 248, 1200, 248);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(-1, 248)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(248);
}
@Test(expected = IllegalArgumentException.class)
public void testNegativeMaxBufferSize() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, -1, 1200, 248);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, -1)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(248);
}
@Test(expected = IllegalArgumentException.class)
public void testMinGreaterThanMaxBufferSize() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 49, 1200, 248);
+ testBufferDebloater()
+ .withDebloatTarget(1200)
+ .withBufferSize(50, 49)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(248);
}
@Test(expected = IllegalArgumentException.class)
public void testNegativeConsumptionTime() {
- testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, -1, 248);
- }
-
- private BufferDebloater testBufferSizeCalculation(
- int numberOfGates,
- List<Integer> numberOfBuffersInUse,
- long throughput,
- long minBufferSize,
- long maxBufferSize,
- int consumptionTime,
- int expectedBufferSize) {
- TestBufferSizeInputGate[] inputGates = new TestBufferSizeInputGate[numberOfGates];
- for (int i = 0; i < numberOfGates; i++) {
- inputGates[i] = new TestBufferSizeInputGate(numberOfBuffersInUse.get(i));
- }
-
- BufferDebloater bufferDebloater =
- new BufferDebloater(
- new Configuration()
- .set(BUFFER_DEBLOAT_ENABLED, true)
- .set(BUFFER_DEBLOAT_TARGET, Duration.ofMillis(consumptionTime))
- .set(
- MEMORY_SEGMENT_SIZE,
- MemorySize.parse("" + maxBufferSize, BYTES))
- .set(
- MIN_MEMORY_SEGMENT_SIZE,
- MemorySize.parse("" + minBufferSize, BYTES)),
- inputGates);
-
- // when: Buffer size is calculated.
- bufferDebloater.recalculateBufferSize(throughput);
-
- // then: Buffer size is in all gates should be as expected.
- for (int i = 0; i < numberOfGates; i++) {
- assertThat(inputGates[i].lastBufferSize, is(expectedBufferSize));
- }
-
- return bufferDebloater;
+ testBufferDebloater()
+ .withDebloatTarget(-1)
+ .withBufferSize(50, 1100)
+ .withNumberOfBuffersInUse(asList(3, 5, 8))
+ .withThroughput(3333)
+ .expectBufferSize(248);
}
private static class TestBufferSizeInputGate extends MockInputGate {
@@ -165,4 +183,69 @@ public class BufferDebloaterTest extends TestLogger {
lastBufferSize = bufferSize;
}
}
+
+ public static BufferDebloaterTestBuilder testBufferDebloater() {
+ return new BufferDebloaterTestBuilder();
+ }
+
+ private static class BufferDebloaterTestBuilder {
+ private List<Integer> numberOfBuffersInUse;
+ private long throughput;
+ private long minBufferSize;
+ private long maxBufferSize;
+ private int debloatTarget;
+
+ public BufferDebloaterTestBuilder withNumberOfBuffersInUse(
+ List<Integer> numberOfBuffersInUse) {
+ this.numberOfBuffersInUse = numberOfBuffersInUse;
+ return this;
+ }
+
+ public BufferDebloaterTestBuilder withThroughput(long throughput) {
+ this.throughput = throughput;
+ return this;
+ }
+
+ public BufferDebloaterTestBuilder withBufferSize(long minBufferSize, long maxBufferSize) {
+ this.minBufferSize = minBufferSize;
+ this.maxBufferSize = maxBufferSize;
+ return this;
+ }
+
+ public BufferDebloaterTestBuilder withDebloatTarget(int debloatTarget) {
+ this.debloatTarget = debloatTarget;
+ return this;
+ }
+
+ public BufferDebloater expectBufferSize(int expectedBufferSize) {
+ int numberOfGates = numberOfBuffersInUse.size();
+ TestBufferSizeInputGate[] inputGates = new TestBufferSizeInputGate[numberOfGates];
+ for (int i = 0; i < numberOfGates; i++) {
+ inputGates[i] = new TestBufferSizeInputGate(numberOfBuffersInUse.get(i));
+ }
+
+ BufferDebloater bufferDebloater =
+ new BufferDebloater(
+ new Configuration()
+ .set(BUFFER_DEBLOAT_ENABLED, true)
+ .set(BUFFER_DEBLOAT_TARGET, Duration.ofMillis(debloatTarget))
+ .set(
+ MEMORY_SEGMENT_SIZE,
+ MemorySize.parse("" + maxBufferSize, BYTES))
+ .set(
+ MIN_MEMORY_SEGMENT_SIZE,
+ MemorySize.parse("" + minBufferSize, BYTES)),
+ inputGates);
+
+ // when: Buffer size is calculated.
+ bufferDebloater.recalculateBufferSize(throughput);
+
+ // then: Buffer size is in all gates should be as expected.
+ for (int i = 0; i < numberOfGates; i++) {
+ assertThat(inputGates[i].lastBufferSize, is(expectedBufferSize));
+ }
+
+ return bufferDebloater;
+ }
+ }
}