You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/04/16 14:12:09 UTC
[spark] branch master updated: [SPARK-27397][CORE] Take care of
OpenJ9 JVM in Spark
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 257d01a [SPARK-27397][CORE] Take care of OpenJ9 JVM in Spark
257d01a is described below
commit 257d01a6b8241c244b932104d2e7c6817d530fd9
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
AuthorDate: Tue Apr 16 09:11:47 2019 -0500
[SPARK-27397][CORE] Take care of OpenJ9 JVM in Spark
## What changes were proposed in this pull request?
This PR supports `OpenJ9` in addition to `IBM JDK` and `OpenJDK` in Spark by handling `System.getProperty("java.vendor") = "Eclipse OpenJ9"`.
In `inferDefaultMemory()` and `getKrb5LoginModuleName()`, this PR uses non `IBM` way.
```
$ ~/jdk-11.0.2+9_openj9-0.12.1/bin/jshell
| Welcome to JShell -- Version 11.0.2
| For an introduction type: /help intro
jshell> System.out.println(System.getProperty("java.vendor"))
Eclipse OpenJ9
jshell> System.out.println(System.getProperty("java.vm.info"))
JRE 11 Linux amd64-64-Bit Compressed References 20190204_127 (JIT enabled, AOT enabled)
OpenJ9 - 90dd8cb40
OMR - d2f4534b
JCL - 289c70b6844 based on jdk-11.0.2+9
jshell> System.out.println(Class.forName("com.ibm.lang.management.OperatingSystemMXBean").getDeclaredMethod("getTotalPhysicalMemory"))
public abstract long com.ibm.lang.management.OperatingSystemMXBean.getTotalPhysicalMemory()
jshell> System.out.println(Class.forName("com.sun.management.OperatingSystemMXBean").getDeclaredMethod("getTotalPhysicalMemorySize"))
public abstract long com.sun.management.OperatingSystemMXBean.getTotalPhysicalMemorySize()
jshell> System.out.println(Class.forName("com.ibm.security.auth.module.Krb5LoginModule"))
| Exception java.lang.ClassNotFoundException: com.ibm.security.auth.module.Krb5LoginModule
| at Class.forNameImpl (Native Method)
| at Class.forName (Class.java:339)
| at (#1:1)
jshell> System.out.println(Class.forName("com.sun.security.auth.module.Krb5LoginModule"))
class com.sun.security.auth.module.Krb5LoginModule
```
## How was this patch tested?
Existing test suites
Manual testing with OpenJ9.
Closes #24308 from kiszk/SPARK-27397.
Authored-by: Kazuaki Ishizaki <is...@jp.ibm.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../scala/org/apache/spark/util/SizeEstimator.scala | 5 +++--
.../apache/spark/launcher/CommandBuilderUtils.java | 20 --------------------
2 files changed, 3 insertions(+), 22 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index 4837b01..e09f1fc 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -131,8 +131,9 @@ object SizeEstimator extends Logging {
return System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY).toBoolean
}
- // java.vm.info provides compressed ref info for IBM JDKs
- if (System.getProperty("java.vendor").contains("IBM")) {
+ // java.vm.info provides compressed ref info for IBM and OpenJ9 JDKs
+ val javaVendor = System.getProperty("java.vendor")
+ if (javaVendor.contains("IBM") || javaVendor.contains("OpenJ9")) {
return System.getProperty("java.vm.info").contains("Compressed Ref")
}
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index 47d2f8e..172fb8c 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -31,11 +31,6 @@ class CommandBuilderUtils {
static final String DEFAULT_PROPERTIES_FILE = "spark-defaults.conf";
static final String ENV_SPARK_HOME = "SPARK_HOME";
- /** The set of known JVM vendors. */
- enum JavaVendor {
- Oracle, IBM, OpenJDK, Unknown
- }
-
/** Returns whether the given string is null or empty. */
static boolean isEmpty(String s) {
return s == null || s.isEmpty();
@@ -112,21 +107,6 @@ class CommandBuilderUtils {
return os.startsWith("Windows");
}
- /** Returns an enum value indicating whose JVM is being used. */
- static JavaVendor getJavaVendor() {
- String vendorString = System.getProperty("java.vendor");
- if (vendorString.contains("Oracle")) {
- return JavaVendor.Oracle;
- }
- if (vendorString.contains("IBM")) {
- return JavaVendor.IBM;
- }
- if (vendorString.contains("OpenJDK")) {
- return JavaVendor.OpenJDK;
- }
- return JavaVendor.Unknown;
- }
-
/**
* Updates the user environment, appending the given pathList to the existing value of the given
* environment variable (or setting it if it hasn't yet been set).
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org