You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2020/11/18 05:15:27 UTC
[hadoop] branch trunk updated: MAPREDUCE-7305. [JDK 11]
TestMRJobsWithProfiler fails. (#2463)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2045a9d MAPREDUCE-7305. [JDK 11] TestMRJobsWithProfiler fails. (#2463)
2045a9d is described below
commit 2045a9df1341523d02987116dba5847317744c52
Author: Akira Ajisaka <aa...@apache.org>
AuthorDate: Wed Nov 18 14:15:11 2020 +0900
MAPREDUCE-7305. [JDK 11] TestMRJobsWithProfiler fails. (#2463)
---
.../mapreduce/v2/TestMRJobsWithProfiler.java | 30 +++++++++++++++++-----
1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
index 9fa8828..fa0c7d6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -43,6 +44,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assume.assumeFalse;
+
public class TestMRJobsWithProfiler {
private static final Logger LOG =
@@ -107,6 +110,8 @@ public class TestMRJobsWithProfiler {
@Test (timeout = 150000)
public void testDefaultProfiler() throws Exception {
+ assumeFalse("The hprof agent has been removed since Java 9. Skipping.",
+ Shell.isJavaVersionAtLeast(9));
LOG.info("Starting testDefaultProfiler");
testProfilerInternal(true);
}
@@ -132,13 +137,21 @@ public class TestMRJobsWithProfiler {
sleepConf.setProfileTaskRange(false, String.valueOf(PROFILED_TASK_ID));
if (!useDefault) {
- // use hprof for map to profile.out
- sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
- "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,"
- + "file=%s");
+ if (Shell.isJavaVersionAtLeast(9)) {
+ // use JDK Flight Recorder
+ sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
+ "-XX:StartFlightRecording=dumponexit=true,filename=%s");
+ sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS,
+ "-XX:StartFlightRecording=dumponexit=true,filename=%s");
+ } else {
+ // use hprof for map to profile.out
+ sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,"
+ + "file=%s");
- // use Xprof for reduce to stdout
- sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof");
+ // use Xprof for reduce to stdout
+ sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof");
+ }
}
sleepJob.setConf(sleepConf);
@@ -211,6 +224,11 @@ public class TestMRJobsWithProfiler {
Assert.assertEquals(4, taLogDirs.size()); // all 4 attempts found
+ // Skip checking the contents because the JFR dumps binary files
+ if (Shell.isJavaVersionAtLeast(9)) {
+ return;
+ }
+
for (Map.Entry<TaskAttemptID,Path> dirEntry : taLogDirs.entrySet()) {
final TaskAttemptID tid = dirEntry.getKey();
final Path profilePath = new Path(dirEntry.getValue(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org