You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2021/10/12 13:54:04 UTC

[flink] 02/03: [refactor][streaming] Builder for parameters of BufferDebloaterTest

This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 0300df70d95cf62c33781f1f0f8de74a54ed030a
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Mon Oct 11 18:35:23 2021 +0200

    [refactor][streaming] Builder for parameters of  BufferDebloaterTest
---
 .../tasks/bufferdebloat/BufferDebloaterTest.java   | 179 +++++++++++++++------
 1 file changed, 131 insertions(+), 48 deletions(-)

diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
index 4dca4c2..c76e4ef 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/bufferdebloat/BufferDebloaterTest.java
@@ -45,29 +45,54 @@ public class BufferDebloaterTest extends TestLogger {
     @Test
     public void testZeroBuffersInUse() {
         // if the gate returns the zero buffers in use it should be transformed to 1.
-        testBufferSizeCalculation(3, asList(0, 1, 0), 3333, 50, 2400, 1000, 1111);
+        testBufferDebloater()
+                .withDebloatTarget(1000)
+                .withBufferSize(50, 2400)
+                .withNumberOfBuffersInUse(asList(0, 1, 0))
+                .withThroughput(3333)
+                .expectBufferSize(1111);
     }
 
     @Test
     public void testCorrectBufferSizeCalculation() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, 1200, 249);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, 1100)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(249);
     }
 
     @Test
     public void testCalculatedBufferSizeLessThanMin() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 250, 1100, 1200, 250);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(250, 1100)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(250);
     }
 
     @Test
     public void testCalculatedBufferSizeForThroughputZero() {
         // When the throughput is zero then min buffer size will be taken.
-        testBufferSizeCalculation(3, asList(3, 5, 8), 0, 50, 1100, 1200, 50);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, 1100)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(0)
+                .expectBufferSize(50);
     }
 
     @Test
     public void testConfiguredConsumptionTimeIsTooLow() {
         // When the consumption time is low then min buffer size will be taken.
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, 7, 50);
+        testBufferDebloater()
+                .withDebloatTarget(7)
+                .withBufferSize(50, 1100)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(50);
     }
 
     @Test
@@ -75,7 +100,12 @@ public class BufferDebloaterTest extends TestLogger {
         // New calculated buffer size should be more than max value it means that we should take max
         // value which means that no updates should happen(-1 means that we take the initial value)
         // because the old value equal to new value.
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 248, 1200, -1);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, 248)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(-1);
     }
 
     @Test
@@ -83,64 +113,52 @@ public class BufferDebloaterTest extends TestLogger {
         // New calculated buffer size should be a little less than current value(or max value which
         // is the same) it means that no updates should happen(-1 means that we take the initial
         // value) because the new value is not so different from the old one.
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 250, 1200, -1);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, 250)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(-1);
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testNegativeMinBufferSize() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, -1, 248, 1200, 248);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(-1, 248)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(248);
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testNegativeMaxBufferSize() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, -1, 1200, 248);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, -1)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(248);
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testMinGreaterThanMaxBufferSize() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 49, 1200, 248);
+        testBufferDebloater()
+                .withDebloatTarget(1200)
+                .withBufferSize(50, 49)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(248);
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testNegativeConsumptionTime() {
-        testBufferSizeCalculation(3, asList(3, 5, 8), 3333, 50, 1100, -1, 248);
-    }
-
-    private BufferDebloater testBufferSizeCalculation(
-            int numberOfGates,
-            List<Integer> numberOfBuffersInUse,
-            long throughput,
-            long minBufferSize,
-            long maxBufferSize,
-            int consumptionTime,
-            int expectedBufferSize) {
-        TestBufferSizeInputGate[] inputGates = new TestBufferSizeInputGate[numberOfGates];
-        for (int i = 0; i < numberOfGates; i++) {
-            inputGates[i] = new TestBufferSizeInputGate(numberOfBuffersInUse.get(i));
-        }
-
-        BufferDebloater bufferDebloater =
-                new BufferDebloater(
-                        new Configuration()
-                                .set(BUFFER_DEBLOAT_ENABLED, true)
-                                .set(BUFFER_DEBLOAT_TARGET, Duration.ofMillis(consumptionTime))
-                                .set(
-                                        MEMORY_SEGMENT_SIZE,
-                                        MemorySize.parse("" + maxBufferSize, BYTES))
-                                .set(
-                                        MIN_MEMORY_SEGMENT_SIZE,
-                                        MemorySize.parse("" + minBufferSize, BYTES)),
-                        inputGates);
-
-        // when: Buffer size is calculated.
-        bufferDebloater.recalculateBufferSize(throughput);
-
-        // then: Buffer size is in all gates should be as expected.
-        for (int i = 0; i < numberOfGates; i++) {
-            assertThat(inputGates[i].lastBufferSize, is(expectedBufferSize));
-        }
-
-        return bufferDebloater;
+        testBufferDebloater()
+                .withDebloatTarget(-1)
+                .withBufferSize(50, 1100)
+                .withNumberOfBuffersInUse(asList(3, 5, 8))
+                .withThroughput(3333)
+                .expectBufferSize(248);
     }
 
     private static class TestBufferSizeInputGate extends MockInputGate {
@@ -165,4 +183,69 @@ public class BufferDebloaterTest extends TestLogger {
             lastBufferSize = bufferSize;
         }
     }
+
+    public static BufferDebloaterTestBuilder testBufferDebloater() {
+        return new BufferDebloaterTestBuilder();
+    }
+
+    private static class BufferDebloaterTestBuilder {
+        private List<Integer> numberOfBuffersInUse;
+        private long throughput;
+        private long minBufferSize;
+        private long maxBufferSize;
+        private int debloatTarget;
+
+        public BufferDebloaterTestBuilder withNumberOfBuffersInUse(
+                List<Integer> numberOfBuffersInUse) {
+            this.numberOfBuffersInUse = numberOfBuffersInUse;
+            return this;
+        }
+
+        public BufferDebloaterTestBuilder withThroughput(long throughput) {
+            this.throughput = throughput;
+            return this;
+        }
+
+        public BufferDebloaterTestBuilder withBufferSize(long minBufferSize, long maxBufferSize) {
+            this.minBufferSize = minBufferSize;
+            this.maxBufferSize = maxBufferSize;
+            return this;
+        }
+
+        public BufferDebloaterTestBuilder withDebloatTarget(int debloatTarget) {
+            this.debloatTarget = debloatTarget;
+            return this;
+        }
+
+        public BufferDebloater expectBufferSize(int expectedBufferSize) {
+            int numberOfGates = numberOfBuffersInUse.size();
+            TestBufferSizeInputGate[] inputGates = new TestBufferSizeInputGate[numberOfGates];
+            for (int i = 0; i < numberOfGates; i++) {
+                inputGates[i] = new TestBufferSizeInputGate(numberOfBuffersInUse.get(i));
+            }
+
+            BufferDebloater bufferDebloater =
+                    new BufferDebloater(
+                            new Configuration()
+                                    .set(BUFFER_DEBLOAT_ENABLED, true)
+                                    .set(BUFFER_DEBLOAT_TARGET, Duration.ofMillis(debloatTarget))
+                                    .set(
+                                            MEMORY_SEGMENT_SIZE,
+                                            MemorySize.parse("" + maxBufferSize, BYTES))
+                                    .set(
+                                            MIN_MEMORY_SEGMENT_SIZE,
+                                            MemorySize.parse("" + minBufferSize, BYTES)),
+                            inputGates);
+
+            // when: Buffer size is calculated.
+            bufferDebloater.recalculateBufferSize(throughput);
+
+            // then: Buffer size is in all gates should be as expected.
+            for (int i = 0; i < numberOfGates; i++) {
+                assertThat(inputGates[i].lastBufferSize, is(expectedBufferSize));
+            }
+
+            return bufferDebloater;
+        }
+    }
 }