You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2023/10/20 09:18:06 UTC
[hadoop] branch branch-3.3 updated: MAPREDUCE-7449: Add add-opens flag to container launch commands on JDK17 nodes (#5935)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 2684221653d9 MAPREDUCE-7449: Add add-opens flag to container launch commands on JDK17 nodes (#5935)
2684221653d9 is described below
commit 2684221653d9ee9a08aabf0e0e7bd29eabb6663d
Author: Benjamin Teke <br...@users.noreply.github.com>
AuthorDate: Fri Oct 20 11:17:59 2023 +0200
MAPREDUCE-7449: Add add-opens flag to container launch commands on JDK17 nodes (#5935)
Contributed by Benjamin Teke
---
.../mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java | 5 +++--
.../src/main/java/org/apache/hadoop/mapred/JobConf.java | 8 ++++++++
.../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 9 +++++++++
.../src/main/resources/mapred-default.xml | 12 ++++++++++++
.../src/main/java/org/apache/hadoop/mapred/YARNRunner.java | 7 +++++++
.../org/apache/hadoop/yarn/api/ApplicationConstants.java | 7 +++++++
.../hadoop/yarn/applications/distributedshell/Client.java | 2 ++
.../containermanager/launcher/ContainerLaunch.java | 7 +++++++
.../containermanager/launcher/TestContainerLaunch.java | 9 ++++++---
9 files changed, 61 insertions(+), 5 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index f00ff281f305..e22bd44ee905 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -52,6 +52,7 @@ public class TestMapReduceChildJVM {
MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
Configuration conf = new Configuration();
conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT, false);
Job job = app.submit(conf);
app.waitForState(job, JobState.SUCCEEDED);
app.verifyCompleted();
@@ -123,7 +124,7 @@ public class TestMapReduceChildJVM {
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN " +
- " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
+ " -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
" -Dlog4j.configuration=container-log4j.properties" +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
@@ -165,7 +166,7 @@ public class TestMapReduceChildJVM {
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN " +
- " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
+ " -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
" -Dlog4j.configuration=" + testLogPropertieFile +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index db6138fae5a0..00d138223601 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2207,6 +2208,13 @@ public class JobConf extends Configuration {
javaOpts += " " + xmxArg;
}
+ // JDK17 support: automatically add --add-opens=java.base/java.lang=ALL-UNNAMED
+ // so the tasks can launch on a JDK17 node.
+ if (getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT,
+ MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) {
+ javaOpts += " " + ApplicationConstants.JVM_ADD_OPENS_VAR;
+ }
+
return javaOpts;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 15d57a6746b1..c26f8d3a1e9e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -305,6 +305,15 @@ public interface MRJobConfig {
"os.name,os.version,java.home,java.runtime.version,java.vendor," +
"java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name";
+ /*
+ * Flag to indicate whether JDK17's required add-opens flags should be added to MR AM and
+ * map/reduce containers regardless of the user specified java opts.
+ */
+ public static final String MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT =
+ "mapreduce.jvm.add-opens-as-default";
+
+ public static final boolean MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT = true;
+
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
public static final int DEFAULT_IO_SORT_FACTOR = 10;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 6fe49400f0ca..097804fdb812 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1811,6 +1811,18 @@
<description>Comma-delimited list of system properties to log on mapreduce JVM start</description>
</property>
+<property>
+ <name>mapreduce.jvm.add-opens-as-default</name>
+ <value>true</value>
+ <description>Since on JDK17 it's no longer possible to use the reflection API to
+ access non-public fields and methods add-opens flags should be added to MR AM
+ and map/reduce containers regardless of the user specified java opts. Setting
+ this to true will add the flags to the container launch commands on nodes with
+ JDK17 or higher. Defaults to true, but the setting has no effect on nodes using
+ JDK16 and before.
+ </description>
+</property>
+
<!-- jobhistory properties -->
<property>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 0fb3f77651e9..18f9286e2424 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -485,6 +485,13 @@ public class YARNRunner implements ClientProtocol {
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
+ // JDK17 support: automatically add --add-opens=java.base/java.lang=ALL-UNNAMED
+ // so the tasks can launch on a JDK17 node.
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT,
+ MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) {
+ vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR);
+ }
+
if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE,
MRJobConfig.DEFAULT_MR_AM_PROFILE)) {
final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 533eaddabb4f..31ba5a9200c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -77,6 +77,13 @@ public interface ApplicationConstants {
String APPLICATION_WEB_PROXY_BASE_ENV =
"APPLICATION_WEB_PROXY_BASE";
+ /**
+ * The environmental variable for JDK17's add-opens workaround. This
+ * should be replaced either a correctly formatted add-opens option if JDK17 is used
+ * or an empty string if not on container launch.
+ */
+ String JVM_ADD_OPENS_VAR = "<ADD_OPENS>";
+
/**
* The temporary environmental variable for container log directory. This
* should be replaced by real container log directory on container launch.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 212d1957aec9..f9fe575976dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -953,6 +953,8 @@ public class Client {
vargs.add("\"" + Environment.JAVA_HOME.$$() + "/bin/java\"");
// Set Xmx based on am memory size
vargs.add("-Xmx" + amMemory + "m");
+ // JDK17 support
+ vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR);
// Set class name
vargs.add(appMasterMainClass);
// Set params for Application Master
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index e864c14ad709..47220de329f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -168,6 +168,13 @@ public class ContainerLaunch implements Callable<Integer> {
var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
File.pathSeparator);
+ if (Shell.isJavaVersionAtLeast(17)) {
+ var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR,
+ "--add-opens=java.base/java.lang=ALL-UNNAMED");
+ } else {
+ var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, "");
+ }
+
// replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
// as %VAR% and on Linux replaced as "$VAR"
if (Shell.WINDOWS) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index a9bcef77c396..9482c4500ea2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -574,18 +574,21 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
+ Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
+ ApplicationConstants.CLASS_PATH_SEPARATOR
+ Apps.crossPlatformify("HADOOP_LOG_HOME")
- + ApplicationConstants.LOG_DIR_EXPANSION_VAR;
+ + ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ + " " + ApplicationConstants.JVM_ADD_OPENS_VAR;
String res = ContainerLaunch.expandEnvironment(input, logPath);
+ String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ?
+ "--add-opens=java.base/java.lang=ALL-UNNAMED" : "";
if (Shell.WINDOWS) {
Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
+ "%HADOOP_HOME%/share/hadoop/common/lib/*;"
- + "%HADOOP_LOG_HOME%/nm/container/logs", res);
+ + "%HADOOP_LOG_HOME%/nm/container/logs" + " " + expectedAddOpens, res);
} else {
Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
+ "$HADOOP_HOME/share/hadoop/common/lib/*:"
- + "$HADOOP_LOG_HOME/nm/container/logs", res);
+ + "$HADOOP_LOG_HOME/nm/container/logs" + " " + expectedAddOpens, res);
}
System.out.println(res);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org