You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2020/02/18 16:26:54 UTC
[flink] branch master updated (77fe6b4 -> e3d5820)
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 77fe6b4 [FLINK-16136][yarn][tests] Increase disk space limit for all YARN tests
new e3da52f [FLINK-14086][core] Add OperatingArchitecture Enum
new 4a361e1 [FLINK-14086][core] (follow-up) Clean-up / unify processor and memory architecture enums
new e3d5820 [FLINK-15473][core] Add ppc64le to the list of known processor architectures.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/flink/util/MemoryArchitecture.java | 77 ----------
.../apache/flink/util/ProcessorArchitecture.java | 156 +++++++++++++++++++++
...ureTest.java => ProcessorArchitectureTest.java} | 15 +-
.../tests/PrometheusReporterEndToEndITCase.java | 28 +++-
.../network/partition/ResultPartitionFactory.java | 5 +-
5 files changed, 194 insertions(+), 87 deletions(-)
delete mode 100755 flink-core/src/main/java/org/apache/flink/util/MemoryArchitecture.java
create mode 100644 flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
rename flink-core/src/test/java/org/apache/flink/util/{MemoryArchitectureTest.java => ProcessorArchitectureTest.java} (66%)
mode change 100755 => 100644
[flink] 01/03: [FLINK-14086][core] Add OperatingArchitecture Enum
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e3da52fa9bed72a611bc174913da1df4452c34e0
Author: wangxiyuan <wa...@huawei.com>
AuthorDate: Wed Sep 25 15:15:21 2019 +0800
[FLINK-14086][core] Add OperatingArchitecture Enum
---
...rchitecture.java => ProcessorArchitecture.java} | 50 +++++++++++++++-------
...ureTest.java => ProcessorArchitectureTest.java} | 10 +++--
.../tests/PrometheusReporterEndToEndITCase.java | 42 ++++++++++++++++--
.../network/partition/ResultPartitionFactory.java | 4 +-
4 files changed, 82 insertions(+), 24 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/util/MemoryArchitecture.java b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
old mode 100755
new mode 100644
similarity index 63%
rename from flink-core/src/main/java/org/apache/flink/util/MemoryArchitecture.java
rename to flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
index 87451d4..47e3094
--- a/flink-core/src/main/java/org/apache/flink/util/MemoryArchitecture.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
@@ -26,10 +26,16 @@ import java.util.List;
* Note that this might be different than the actual operating system's architecture, for example
* when installing a 32 bit JRE in a 64 bit OS.
*/
-public enum MemoryArchitecture {
+public enum ProcessorArchitecture {
+ /**
+ * X86 platform.
+ */
+ X86,
- // constants here start with an underscore because Java identifier cannot start with a
- // numeric character and alternatives like 'BIT_64' are not as readable
+ /**
+ * arm platform.
+ */
+ ARM,
/**
* 32 bit memory address size.
@@ -46,18 +52,25 @@ public enum MemoryArchitecture {
*/
UNKNOWN;
- // ------------------------------------------------------------------------
- private static final MemoryArchitecture current = getInternal();
+ private static final ProcessorArchitecture arch = readArchFromSystemProperties();
+ private static final ProcessorArchitecture size = readSizeFromSystemProperties();
- /**
- * Gets the processor architecture of this process.
- */
- public static MemoryArchitecture get() {
- return current;
+ private static ProcessorArchitecture readArchFromSystemProperties() {
+ final List<String> namesX86 = Arrays.asList("amd64", "x86_64", "x86", "i386", "i486", "i586", "i686");
+ final List<String> namesArm = Arrays.asList("arm", "aarch64");
+ final String arch = System.getProperty("os.arch");
+
+ if (namesX86.contains(arch)) {
+ return X86;
+ } else if (namesArm.contains(arch)) {
+ return ARM;
+ } else {
+ return UNKNOWN;
+ }
}
- private static MemoryArchitecture getInternal() {
+ private static ProcessorArchitecture readSizeFromSystemProperties() {
// putting these into the method to avoid having objects on the heap that are not needed
// any more after initialization
final List<String> names64bit = Arrays.asList("amd64", "x86_64", "aarch64");
@@ -66,12 +79,19 @@ public enum MemoryArchitecture {
if (names64bit.contains(arch)) {
return _64_BIT;
- }
- else if (names32bit.contains(arch)) {
+ } else if (names32bit.contains(arch)) {
return _32_BIT;
- }
- else {
+ } else {
return UNKNOWN;
}
}
+
+ public static ProcessorArchitecture getCurrentOperatingSystemArch() {
+ return arch;
+ }
+
+ public static ProcessorArchitecture getCurrentOperatingSystemSize() {
+ return size;
+ }
+
}
diff --git a/flink-core/src/test/java/org/apache/flink/util/MemoryArchitectureTest.java b/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
old mode 100755
new mode 100644
similarity index 71%
rename from flink-core/src/test/java/org/apache/flink/util/MemoryArchitectureTest.java
rename to flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
index 92c829f..369b913
--- a/flink-core/src/test/java/org/apache/flink/util/MemoryArchitectureTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
@@ -23,14 +23,16 @@ import org.junit.Test;
import static org.junit.Assert.assertNotEquals;
/**
- * Tests for the {@link MemoryArchitecture}.
+ * Tests for the {@link ProcessorArchitecture}.
*/
-public class MemoryArchitectureTest {
+public class ProcessorArchitectureTest {
@Test
public void testArchitectureNotUnknown() {
- final MemoryArchitecture arch = MemoryArchitecture.get();
+ final ProcessorArchitecture arch = ProcessorArchitecture.getCurrentOperatingSystemArch();
+ final ProcessorArchitecture size = ProcessorArchitecture.getCurrentOperatingSystemSize();
- assertNotEquals(MemoryArchitecture.UNKNOWN, arch);
+ assertNotEquals(ProcessorArchitecture.UNKNOWN, arch);
+ assertNotEquals(ProcessorArchitecture.UNKNOWN, size);
}
}
diff --git a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
index 56643b2..c70a5198 100644
--- a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
+++ b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.tests.util.cache.DownloadCache;
import org.apache.flink.tests.util.categories.TravisGroup1;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.OperatingSystem;
+import org.apache.flink.util.ProcessorArchitecture;
import org.apache.flink.util.TestLogger;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
@@ -68,17 +69,52 @@ public class PrometheusReporterEndToEndITCase extends TestLogger {
static {
final String base = "prometheus-" + PROMETHEUS_VERSION + '.';
+ final String os;
+ final String platform;
switch (OperatingSystem.getCurrentOperatingSystem()) {
case MAC_OS:
- PROMETHEUS_FILE_NAME = base + "darwin-amd64";
+ os = "darwin";
break;
case WINDOWS:
- PROMETHEUS_FILE_NAME = base + "windows-amd64";
+ os = "windows";
break;
default:
- PROMETHEUS_FILE_NAME = base + "linux-amd64";
+ os = "linux";
break;
}
+ switch (ProcessorArchitecture.getCurrentOperatingSystemArch()) {
+ case X86:
+ switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
+ case _32_BIT:
+ platform = "386";
+ break;
+ case _64_BIT:
+ platform = "amd64";
+ break;
+ default:
+ platform = "Unknown";
+ break;
+ }
+ break;
+ case ARM:
+ switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
+ case _32_BIT:
+ platform = "armv7";
+ break;
+ case _64_BIT:
+ platform = "arm64";
+ break;
+ default:
+ platform = "Unknown";
+ break;
+ }
+ break;
+ default:
+ platform = "Unknown";
+ break;
+ }
+
+ PROMETHEUS_FILE_NAME = base + os + "-" + platform;
}
private static final Pattern LOG_REPORTER_PORT_PATTERN = Pattern.compile(".*Started PrometheusReporter HTTP server on port ([0-9]+).*");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
index 2feceac..604164d 100755
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
@@ -28,7 +28,7 @@ import org.apache.flink.runtime.io.network.buffer.BufferPoolFactory;
import org.apache.flink.runtime.io.network.buffer.BufferPoolOwner;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkRuntimeException;
-import org.apache.flink.util.MemoryArchitecture;
+import org.apache.flink.util.ProcessorArchitecture;
import org.apache.flink.util.function.FunctionWithException;
import org.slf4j.Logger;
@@ -220,7 +220,7 @@ public class ResultPartitionFactory {
}
static BoundedBlockingSubpartitionType getBoundedBlockingType() {
- switch (MemoryArchitecture.get()) {
+ switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
case _64_BIT:
return BoundedBlockingSubpartitionType.FILE_MMAP;
case _32_BIT:
[flink] 02/03: [FLINK-14086][core] (follow-up) Clean-up / unify
processor and memory architecture enums
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4a361e17fc5c1cfa957920da8a92556af81ad141
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Tue Feb 18 09:53:16 2020 +0100
[FLINK-14086][core] (follow-up) Clean-up / unify processor and memory architecture enums
This closes #9768
---
.../apache/flink/util/ProcessorArchitecture.java | 142 ++++++++++++++-------
.../flink/util/ProcessorArchitectureTest.java | 13 +-
.../tests/PrometheusReporterEndToEndITCase.java | 34 ++---
.../network/partition/ResultPartitionFactory.java | 3 +-
4 files changed, 118 insertions(+), 74 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
index 47e3094..9b34eca 100644
--- a/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
@@ -19,79 +19,133 @@
package org.apache.flink.util;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/**
- * The memory architecture (32 bit / 64 bit) of the current process.
- * Note that this might be different than the actual operating system's architecture, for example
- * when installing a 32 bit JRE in a 64 bit OS.
+ * The processor architecture of the this system.
+ *
+ * <p>Note that the memory address size might be different than the actual hardware architecture,
+ * due to the installed OS (32bit OS) or when installing a 32 bit JRE in a 64 bit OS.
*/
public enum ProcessorArchitecture {
+
+ /**
+ * The Intel x86 processor architecture.
+ */
+ X86(MemoryAddressSize._32_BIT, "x86", "i386", "i486", "i586", "i686"),
+
/**
- * X86 platform.
+ * The AMD 64 bit processor architecture.
*/
- X86,
+ AMD64(MemoryAddressSize._64_BIT, "amd64", "x86_64"),
/**
- * arm platform.
+ * The ARM 32 bit processor architecture.
*/
- ARM,
+ ARMv7(MemoryAddressSize._32_BIT, "armv7", "arm"),
/**
- * 32 bit memory address size.
+ * The 64 bit ARM processor architecture.
*/
- _32_BIT,
+ AARCH64(MemoryAddressSize._64_BIT, "aarch64"),
+
+ /**
+ * Unknown architecture, could not be determined. This one conservatively assumes 32 bit,
+ * because 64 bit platforms typically support 32 bit memory spaces.
+ */
+ UNKNOWN(MemoryAddressSize._32_BIT, "unknown");
+
+ // ------------------------------------------------------------------------
+
+ private static final ProcessorArchitecture CURRENT = readArchFromSystemProperties();
+
+ private final MemoryAddressSize addressSize;
+
+ private final String name;
+
+ private final List<String> alternativeNames;
+
+ ProcessorArchitecture(MemoryAddressSize addressSize, String name, String... alternativeNames) {
+ this.addressSize = addressSize;
+ this.name = name;
+ this.alternativeNames = Collections.unmodifiableList(Arrays.asList(alternativeNames));
+ }
+
+ /**
+ * Gets the address size of the memory (32 bit, 64 bit).
+ */
+ public MemoryAddressSize getAddressSize() {
+ return addressSize;
+ }
/**
- * 64 bit memory address size.
+ * Gets the primary name of the processor architecture.
+ * The primary name would for example be "x86" or "amd64".
*/
- _64_BIT,
+ public String getArchitectureName() {
+ return name;
+ }
/**
- * Unknown architecture, could not be determined.
+ * Gets the alternative names for the processor architecture.
+ * Alternative names are for example "i586" for "x86", or "x86_64" for "amd64".
*/
- UNKNOWN;
+ public List<String> getAlternativeNames() {
+ return alternativeNames;
+ }
+
+ // ------------------------------------------------------------------------
+ /**
+ * Gets the ProcessorArchitecture of the system running this process.
+ */
+ public static ProcessorArchitecture getProcessorArchitecture() {
+ return CURRENT;
+ }
- private static final ProcessorArchitecture arch = readArchFromSystemProperties();
- private static final ProcessorArchitecture size = readSizeFromSystemProperties();
+ /**
+ * Gets the MemorySize of the ProcessorArchitecture of this process.
+ *
+ * <p>Note that the memory address size might be different than the actual hardware architecture,
+ * due to the installed OS (32bit OS) or when installing a 32 bit JRE in a 64 bit OS.
+ */
+ public static MemoryAddressSize getMemoryAddressSize() {
+ return getProcessorArchitecture().getAddressSize();
+ }
private static ProcessorArchitecture readArchFromSystemProperties() {
- final List<String> namesX86 = Arrays.asList("amd64", "x86_64", "x86", "i386", "i486", "i586", "i686");
- final List<String> namesArm = Arrays.asList("arm", "aarch64");
- final String arch = System.getProperty("os.arch");
-
- if (namesX86.contains(arch)) {
- return X86;
- } else if (namesArm.contains(arch)) {
- return ARM;
- } else {
+ final String sysArchName = System.getProperty("os.arch");
+ if (sysArchName == null) {
return UNKNOWN;
}
- }
- private static ProcessorArchitecture readSizeFromSystemProperties() {
- // putting these into the method to avoid having objects on the heap that are not needed
- // any more after initialization
- final List<String> names64bit = Arrays.asList("amd64", "x86_64", "aarch64");
- final List<String> names32bit = Arrays.asList("x86", "i386", "i486", "i586", "i686");
- final String arch = System.getProperty("os.arch");
-
- if (names64bit.contains(arch)) {
- return _64_BIT;
- } else if (names32bit.contains(arch)) {
- return _32_BIT;
- } else {
- return UNKNOWN;
+ for (ProcessorArchitecture arch : values()) {
+ if (sysArchName.equalsIgnoreCase(arch.name)) {
+ return arch;
+ }
+
+ for (String altName : arch.alternativeNames) {
+ if (sysArchName.equalsIgnoreCase(altName)) {
+ return arch;
+ }
+ }
}
- }
- public static ProcessorArchitecture getCurrentOperatingSystemArch() {
- return arch;
+ return UNKNOWN;
}
- public static ProcessorArchitecture getCurrentOperatingSystemSize() {
- return size;
- }
+ // ------------------------------------------------------------------------
+ /**
+ * The memory address size of the processor.
+ */
+ public enum MemoryAddressSize {
+
+ /** 32 bit memory address size. */
+ _32_BIT,
+
+ /** 64 bit memory address size. */
+ _64_BIT,
+ }
}
diff --git a/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java b/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
index 369b913..9ec2c74 100644
--- a/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/ProcessorArchitectureTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.util;
import org.junit.Test;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
/**
* Tests for the {@link ProcessorArchitecture}.
@@ -29,10 +30,14 @@ public class ProcessorArchitectureTest {
@Test
public void testArchitectureNotUnknown() {
- final ProcessorArchitecture arch = ProcessorArchitecture.getCurrentOperatingSystemArch();
- final ProcessorArchitecture size = ProcessorArchitecture.getCurrentOperatingSystemSize();
-
+ final ProcessorArchitecture arch = ProcessorArchitecture.getProcessorArchitecture();
assertNotEquals(ProcessorArchitecture.UNKNOWN, arch);
- assertNotEquals(ProcessorArchitecture.UNKNOWN, size);
+ }
+
+ @Test
+ public void testNamesNotNull() {
+ final ProcessorArchitecture arch = ProcessorArchitecture.getProcessorArchitecture();
+ assertNotNull(arch.getArchitectureName());
+ assertNotNull(arch.getAlternativeNames());
}
}
diff --git a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
index c70a5198..fe8fe10 100644
--- a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
+++ b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java
@@ -82,32 +82,18 @@ public class PrometheusReporterEndToEndITCase extends TestLogger {
os = "linux";
break;
}
- switch (ProcessorArchitecture.getCurrentOperatingSystemArch()) {
+ switch (ProcessorArchitecture.getProcessorArchitecture()) {
case X86:
- switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
- case _32_BIT:
- platform = "386";
- break;
- case _64_BIT:
- platform = "amd64";
- break;
- default:
- platform = "Unknown";
- break;
- }
+ platform = "386";
break;
- case ARM:
- switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
- case _32_BIT:
- platform = "armv7";
- break;
- case _64_BIT:
- platform = "arm64";
- break;
- default:
- platform = "Unknown";
- break;
- }
+ case AMD64:
+ platform = "amd64";
+ break;
+ case ARMv7:
+ platform = "armv7";
+ break;
+ case AARCH64:
+ platform = "arm64";
break;
default:
platform = "Unknown";
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
index 604164d..e924bd0 100755
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
@@ -220,12 +220,11 @@ public class ResultPartitionFactory {
}
static BoundedBlockingSubpartitionType getBoundedBlockingType() {
- switch (ProcessorArchitecture.getCurrentOperatingSystemSize()) {
+ switch (ProcessorArchitecture.getMemoryAddressSize()) {
case _64_BIT:
return BoundedBlockingSubpartitionType.FILE_MMAP;
case _32_BIT:
return BoundedBlockingSubpartitionType.FILE;
- case UNKNOWN:
default:
LOG.warn("Cannot determine memory architecture. Using pure file-based shuffle.");
return BoundedBlockingSubpartitionType.FILE;
[flink] 03/03: [FLINK-15473][core] Add ppc64le to the list of known
processor architectures.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e3d58209f7aea43756034f9bd7d07a90c41b35ee
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Tue Feb 18 09:58:37 2020 +0100
[FLINK-15473][core] Add ppc64le to the list of known processor architectures.
---
.../src/main/java/org/apache/flink/util/ProcessorArchitecture.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
index 9b34eca..cfa9b5c 100644
--- a/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ProcessorArchitecture.java
@@ -51,6 +51,11 @@ public enum ProcessorArchitecture {
AARCH64(MemoryAddressSize._64_BIT, "aarch64"),
/**
+ * The little-endian mode of the 64 bit Power-PC architecture.
+ */
+ PPC64_LE(MemoryAddressSize._64_BIT, "ppc64le"),
+
+ /**
* Unknown architecture, could not be determined. This one conservatively assumes 32 bit,
* because 64 bit platforms typically support 32 bit memory spaces.
*/