You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by nk...@apache.org on 2019/07/30 10:09:40 UTC
[flink] 01/04: [hotfix][test] Extract common variables
This is an automated email from the ASF dual-hosted git repository.
nkruber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f384449d9afd40228334f255ba1e02474dddd8b6
Author: Gao Yun <yu...@alibaba-inc.com>
AuthorDate: Thu Jul 11 19:48:33 2019 +0800
[hotfix][test] Extract common variables
---
.../TaskManagerHeapSizeCalculationJavaBashTest.java | 20 ++++++++++++++------
.../taskexecutor/NetworkBufferCalculationTest.java | 11 +++++++----
2 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
index 9d93180..44277c5 100755
--- a/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
+++ b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
@@ -76,19 +76,23 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger {
*/
@Test
public void compareNetworkBufShellScriptWithJava() throws Exception {
+ final int totalMemoryInMB = 1000; // 1000MB
+ final long networkBufMin = 64L << 20; // 64MB
+ final long networkBufMax = 1L << 30; // 1GB
+
int managedMemSize = Integer.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue());
float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue();
// manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB()
compareNetworkBufJavaVsScript(
- getConfig(1000, false, 0.1f, 64L << 20, 1L << 30, managedMemSize, managedMemFrac), 0.0f);
+ getConfig(totalMemoryInMB, false, 0.1f, networkBufMin, networkBufMax, managedMemSize, managedMemFrac), 0.0f);
compareNetworkBufJavaVsScript(
- getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, 10 /*MB*/, managedMemFrac), 0.0f);
+ getConfig(totalMemoryInMB, true, 0.1f, networkBufMin, networkBufMax, 10 /*MB*/, managedMemFrac), 0.0f);
compareNetworkBufJavaVsScript(
- getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, managedMemSize, 0.1f), 0.0f);
+ getConfig(totalMemoryInMB, true, 0.1f, networkBufMin, networkBufMax, managedMemSize, 0.1f), 0.0f);
// some automated tests with random (but valid) values
@@ -105,19 +109,23 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger {
*/
@Test
public void compareHeapSizeShellScriptWithJava() throws Exception {
+ final int totalMemoryInMB = 1000; // 1000MB
+ final long networkBufMin = 64L << 20; // 64MB
+ final long networkBufMax = 1L << 30; // 1GB
+
int managedMemSize = Integer.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue());
float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue();
// manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB()
compareHeapSizeJavaVsScript(
- getConfig(1000, false, 0.1f, 64L << 20, 1L << 30, managedMemSize, managedMemFrac), 0.0f);
+ getConfig(totalMemoryInMB, false, 0.1f, networkBufMin, networkBufMax, managedMemSize, managedMemFrac), 0.0f);
compareHeapSizeJavaVsScript(
- getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, 10 /*MB*/, managedMemFrac), 0.0f);
+ getConfig(totalMemoryInMB, true, 0.1f, networkBufMin, networkBufMax, 10 /*MB*/, managedMemFrac), 0.0f);
compareHeapSizeJavaVsScript(
- getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, managedMemSize, 0.1f), 0.0f);
+ getConfig(totalMemoryInMB, true, 0.1f, networkBufMin, networkBufMax, managedMemSize, 0.1f), 0.0f);
// some automated tests with random (but valid) values
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
index 39b9d74..3ad9a2e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
@@ -42,26 +42,29 @@ public class NetworkBufferCalculationTest extends TestLogger {
public void calculateNetworkBufFromHeapSize() {
Configuration config;
+ final long networkBufMin = 64L << 20; // 64MB
+ final long networkBufMax = 1L << 30; // 1GB
+
config = getConfig(
Long.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()),
TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
- 0.1f, 60L << 20, 1L << 30, MemoryType.HEAP);
+ 0.1f, networkBufMin, networkBufMax, MemoryType.HEAP);
assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 900L << 20)); // 900MB
config = getConfig(
Long.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()),
TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
- 0.2f, 60L << 20, 1L << 30, MemoryType.HEAP);
+ 0.2f, networkBufMin, networkBufMax, MemoryType.HEAP);
assertEquals((200L << 20) + 3 /* slightly too many due to floating point imprecision */,
NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 800L << 20)); // 800MB
config = getConfig(10, TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
- 0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
+ 0.1f, networkBufMin, networkBufMax, MemoryType.OFF_HEAP);
assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 890L << 20)); // 890MB
- config = getConfig(0, 0.1f, 0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
+ config = getConfig(0, 0.1f, 0.1f, networkBufMin, networkBufMax, MemoryType.OFF_HEAP);
assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 810L << 20)); // 810MB
}