You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2014/01/17 21:57:54 UTC
svn commit: r1559250 - in
/hadoop/common/branches/YARN-321/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/...
Author: vinodkv
Date: Fri Jan 17 20:57:53 2014
New Revision: 1559250
URL: http://svn.apache.org/r1559250
Log:
YARN-321. Forwarding YARN-321 branch to latest trunk.
Added:
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
- copied unchanged from r1559249, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
Modified:
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
Propchange: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1559007-1559249
Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt?rev=1559250&r1=1559249&r2=1559250&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt Fri Jan 17 20:57:53 2014
@@ -334,6 +334,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
(Chuan Liu via cnauroth)
+ MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params
+ is specified (Gera Shegalov via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1559007-1559249
Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1559250&r1=1559249&r2=1559250&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Fri Jan 17 20:57:53 2014
@@ -212,19 +212,11 @@ public class MapReduceChildJVM {
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
).isIncluded(task.getPartition())) {
- vargs.add(
- String.format(
- conf.getProfileParams(),
- getTaskLogFile(TaskLog.LogName.PROFILE)
- )
- );
- if (task.isMapTask()) {
- vargs.add(conf.get(MRJobConfig.TASK_MAP_PROFILE_PARAMS, ""));
- }
- else {
- vargs.add(conf.get(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, ""));
- }
-
+ final String profileParams = conf.get(task.isMapTask()
+ ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
+ : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
+ vargs.add(String.format(profileParams,
+ getTaskLogFile(TaskLog.LogName.PROFILE)));
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1559250&r1=1559249&r2=1559250&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Fri Jan 17 20:57:53 2014
@@ -603,6 +603,31 @@
</property>
<property>
+ <name>mapreduce.task.profile.params</name>
+ <value></value>
+ <description>JVM profiler parameters used to profile map and reduce task
+ attempts. This string may contain a single format specifier %s that will
+ be replaced by the path to profile.out in the task attempt log directory.
+ To specify different profiling options for map tasks and reduce tasks,
+ more specific parameters mapreduce.task.profile.map.params and
+ mapreduce.task.profile.reduce.params should be used.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.map.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Map-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.reduce.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Reduce-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
<name>mapreduce.task.skip.start.attempts</name>
<value>2</value>
<description> The number of Task attempts AFTER which skip mode
Propchange: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1559007-1559249
Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1559250&r1=1559249&r2=1559250&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Fri Jan 17 20:57:53 2014
@@ -90,6 +90,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
@@ -99,6 +101,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -354,6 +357,12 @@ public class TestClientRedirect {
CancelDelegationTokenRequest request) throws IOException {
return null;
}
+
+ @Override
+ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
+ MoveApplicationAcrossQueuesRequest request) throws YarnException, IOException {
+ return null;
+ }
}
class HistoryService extends AMService implements HSClientProtocol {