You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/10/31 07:48:10 UTC
svn commit: r1195353 [2/2] - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/s...
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java Mon Oct 31 06:48:08 2011
@@ -218,7 +218,8 @@ public class TestHSWebApp {
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
.toString());
- params.put(NM_NODENAME, BuilderUtils.newNodeId("testhost", 2222).toString());
+ params.put(NM_NODENAME,
+ BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
params.put(ENTITY_STRING, "container_10_0001_01_000001");
params.put(APP_OWNER, "owner");
@@ -229,7 +230,8 @@ public class TestHSWebApp {
verify(spyPw).write(
"Logs not available for container_10_0001_01_000001. Aggregation "
+ "may not be complete,"
- + " Check back later or try the nodemanager on testhost:2222");
+ + " Check back later or try the nodemanager on "
+ + MockJobs.NM_HOST + ":" + MockJobs.NM_PORT);
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Mon Oct 31 06:48:08 2011
@@ -23,6 +23,7 @@ import java.lang.reflect.InvocationTarge
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
@@ -37,6 +38,7 @@ import org.apache.hadoop.mapreduce.JobSt
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -47,13 +49,17 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -68,6 +74,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
+import org.apache.hadoop.yarn.util.BuilderUtils;
public class ClientServiceDelegate {
private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
@@ -398,5 +405,52 @@ public class ClientServiceDelegate {
return true;
}
+ public LogParams getLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID)
+ throws YarnRemoteException, IOException {
+ org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+ TypeConverter.toYarn(oldJobID);
+ GetJobReportRequest request =
+ recordFactory.newRecordInstance(GetJobReportRequest.class);
+ request.setJobId(jobId);
-}
+ JobReport report =
+ ((GetJobReportResponse) invoke("getJobReport",
+ GetJobReportRequest.class, request)).getJobReport();
+ if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED,
+ JobState.ERROR).contains(report.getJobState())) {
+ if (oldTaskAttemptID != null) {
+ GetTaskAttemptReportRequest taRequest =
+ recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
+ taRequest.setTaskAttemptId(TypeConverter.toYarn(oldTaskAttemptID));
+ TaskAttemptReport taReport =
+ ((GetTaskAttemptReportResponse) invoke("getTaskAttemptReport",
+ GetTaskAttemptReportRequest.class, taRequest))
+ .getTaskAttemptReport();
+ if (taReport.getContainerId() == null
+ || taReport.getNodeManagerHost() == null) {
+ throw new IOException("Unable to get log information for task: "
+ + oldTaskAttemptID);
+ }
+ return new LogParams(
+ taReport.getContainerId().toString(),
+ taReport.getContainerId().getApplicationAttemptId()
+ .getApplicationId().toString(),
+ BuilderUtils.newNodeId(taReport.getNodeManagerHost(),
+ taReport.getNodeManagerPort()).toString(), report.getUser());
+ } else {
+ if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
+ throw new IOException("Unable to get log information for job: "
+ + oldJobID);
+ }
+ AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1);
+ return new LogParams(
+ amInfo.getContainerId().toString(),
+ amInfo.getAppAttemptId().getApplicationId().toString(),
+ BuilderUtils.newNodeId(amInfo.getNodeManagerHost(),
+ amInfo.getNodeManagerPort()).toString(), report.getUser());
+ }
+ } else {
+ throw new IOException("Cannot get log path for a in-progress job");
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Mon Oct 31 06:48:08 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.TaskT
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
@@ -504,4 +505,10 @@ public class YARNRunner implements Clien
return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion,
clientMethodsHash);
}
+
+ @Override
+ public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID)
+ throws IOException {
+ return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Mon Oct 31 06:48:08 2011
@@ -168,7 +168,7 @@ public class TestClientServiceDelegate {
GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
when(jobReportResponse1.getJobReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user",
- JobState.RUNNING, 0, 0, 0, 0, 0, 0, "anything"));
+ JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
@@ -180,7 +180,7 @@ public class TestClientServiceDelegate {
GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
when(jobReportResponse2.getJobReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user",
- JobState.RUNNING, 0, 0, 0, 0, 0, 0, "anything"));
+ JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Mon Oct 31 06:48:08 2011
@@ -20,13 +20,13 @@ package org.apache.hadoop.mapreduce.v2;
import java.io.File;
import java.io.IOException;
+import java.util.List;
import junit.framework.Assert;
import org.apache.avro.AvroRemoteException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.FailingMapper;
import org.apache.hadoop.SleepJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -35,8 +35,20 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Before;
import org.junit.After;
import org.junit.Test;
@@ -105,6 +117,8 @@ public class TestMRJobsWithHistoryServic
return;
}
+
+
SleepJob sleepJob = new SleepJob();
sleepJob.setConf(mrCluster.getConfig());
// Job with 3 maps and 2 reduces
@@ -113,7 +127,8 @@ public class TestMRJobsWithHistoryServic
job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
job.waitForCompletion(true);
Counters counterMR = job.getCounters();
- ApplicationId appID = TypeConverter.toYarn(job.getJobID()).getAppId();
+ JobId jobId = TypeConverter.toYarn(job.getJobID());
+ ApplicationId appID = jobId.getAppId();
while (true) {
Thread.sleep(1000);
if (mrCluster.getResourceManager().getRMContext().getRMApps()
@@ -126,6 +141,36 @@ public class TestMRJobsWithHistoryServic
LOG.info("CounterHS " + counterHS);
LOG.info("CounterMR " + counterMR);
Assert.assertEquals(counterHS, counterMR);
+
+ MRClientProtocol historyClient = instantiateHistoryProxy();
+ GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
+ gjReq.setJobId(jobId);
+ JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
+ verifyJobReport(jobReport, jobId);
}
+ private void verifyJobReport(JobReport jobReport, JobId jobId) {
+ List<AMInfo> amInfos = jobReport.getAMInfos();
+ Assert.assertEquals(1, amInfos.size());
+ AMInfo amInfo = amInfos.get(0);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(jobId.getAppId(), 1);
+ ContainerId amContainerId = BuilderUtils.newContainerId(appAttemptId, 1);
+ Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId());
+ Assert.assertEquals(amContainerId, amInfo.getContainerId());
+ Assert.assertTrue(jobReport.getSubmitTime() > 0);
+ Assert.assertTrue(jobReport.getStartTime() > 0
+ && jobReport.getStartTime() >= jobReport.getSubmitTime());
+ Assert.assertTrue(jobReport.getFinishTime() > 0
+ && jobReport.getFinishTime() >= jobReport.getStartTime());
+ }
+
+ private MRClientProtocol instantiateHistoryProxy() {
+ final String serviceAddr =
+ mrCluster.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS);
+ final YarnRPC rpc = YarnRPC.create(conf);
+ MRClientProtocol historyClient =
+ (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
+ NetUtils.createSocketAddr(serviceAddr), mrCluster.getConfig());
+ return historyClient;
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Mon Oct 31 06:48:08 2011
@@ -126,6 +126,14 @@ public class ConverterUtils {
return appAttemptId;
}
+ private static ApplicationId toApplicationId(
+ Iterator<String> it) throws NumberFormatException {
+ ApplicationId appId = Records.newRecord(ApplicationId.class);
+ appId.setClusterTimestamp(Long.parseLong(it.next()));
+ appId.setId(Integer.parseInt(it.next()));
+ return appId;
+ }
+
public static String toString(ContainerId cId) {
return cId.toString();
}
@@ -178,4 +186,18 @@ public class ConverterUtils {
}
}
+ public static ApplicationId toApplicationId(
+ String appIdStr) {
+ Iterator<String> it = _split(appIdStr).iterator();
+ if (!it.next().equals(APPLICATION_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ApplicationId prefix: "
+ + appIdStr);
+ }
+ try {
+ return toApplicationId(it);
+ } catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid AppAttemptId: "
+ + appIdStr, n);
+ }
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Mon Oct 31 06:48:08 2011
@@ -140,7 +140,7 @@ public class LogAggregationService exten
}
super.stop();
}
-
+
/**
* Constructs the full filename for an application's log file per node.
* @param remoteRootLogDir
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java Mon Oct 31 06:48:08 2011
@@ -50,6 +50,7 @@ public class LogDumper extends Configure
private static final String CONTAINER_ID_OPTION = "containerId";
private static final String APPLICATION_ID_OPTION = "applicationId";
private static final String NODE_ADDRESS_OPTION = "nodeAddress";
+ private static final String APP_OWNER_OPTION = "appOwner";
@Override
public int run(String[] args) throws Exception {
@@ -58,6 +59,7 @@ public class LogDumper extends Configure
opts.addOption(APPLICATION_ID_OPTION, true, "ApplicationId");
opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId");
opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress");
+ opts.addOption(APP_OWNER_OPTION, true, "AppOwner");
if (args.length < 1) {
HelpFormatter formatter = new HelpFormatter();
@@ -69,11 +71,13 @@ public class LogDumper extends Configure
String appIdStr = null;
String containerIdStr = null;
String nodeAddress = null;
+ String appOwner = null;
try {
CommandLine commandLine = parser.parse(opts, args, true);
appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
+ appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
} catch (ParseException e) {
System.out.println("options parsing failed: " + e.getMessage());
@@ -96,8 +100,11 @@ public class LogDumper extends Configure
DataOutputStream out = new DataOutputStream(System.out);
+ if (appOwner == null || appOwner.isEmpty()) {
+ appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+ }
if (containerIdStr == null && nodeAddress == null) {
- dumpAllContainersLogs(appId, out);
+ dumpAllContainersLogs(appId, appOwner, out);
} else if ((containerIdStr == null && nodeAddress != null)
|| (containerIdStr != null && nodeAddress == null)) {
System.out.println("ContainerId or NodeAddress cannot be null!");
@@ -113,7 +120,7 @@ public class LogDumper extends Configure
LogAggregationService.getRemoteNodeLogFileForApp(
remoteRootLogDir,
appId,
- UserGroupInformation.getCurrentUser().getShortUserName(),
+ appOwner,
ConverterUtils.toNodeId(nodeAddress),
getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX)));
@@ -123,6 +130,21 @@ public class LogDumper extends Configure
return 0;
}
+ public void dumpAContainersLogs(String appId, String containerId,
+ String nodeId, String jobOwner) throws IOException {
+ Path remoteRootLogDir =
+ new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
+ String suffix = LogAggregationService.getRemoteNodeLogDirSuffix(getConf());
+ AggregatedLogFormat.LogReader reader =
+ new AggregatedLogFormat.LogReader(getConf(),
+ LogAggregationService.getRemoteNodeLogFileForApp(remoteRootLogDir,
+ ConverterUtils.toApplicationId(appId), jobOwner,
+ ConverterUtils.toNodeId(nodeId), suffix));
+ DataOutputStream out = new DataOutputStream(System.out);
+ dumpAContainerLogs(containerId, reader, out);
+ }
+
private int dumpAContainerLogs(String containerIdStr,
AggregatedLogFormat.LogReader reader, DataOutputStream out)
throws IOException {
@@ -152,13 +174,12 @@ public class LogDumper extends Configure
return 0;
}
- private void
- dumpAllContainersLogs(ApplicationId appId, DataOutputStream out)
- throws IOException {
+ private void dumpAllContainersLogs(ApplicationId appId, String appOwner,
+ DataOutputStream out) throws IOException {
Path remoteRootLogDir =
new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
- String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ String user = appOwner;
String logDirSuffix =
getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX);
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Oct 31 06:48:08 2011
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.serve
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
+import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
@@ -4822,6 +4823,13 @@ public class JobTracker implements MRCon
return secretManager.renewToken(token, user);
}
+ @Override
+ public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
+ org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException("Not supported by JobTracker");
+ }
+
JobACLsManager getJobACLsManager() {
return aclsManager.getJobACLsManager();
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1195353&r1=1195352&r2=1195353&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Mon Oct 31 06:48:08 2011
@@ -54,6 +54,7 @@ import org.apache.hadoop.mapreduce.filec
import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.State;
@@ -812,4 +813,11 @@ public class LocalJobRunner implements C
) throws IOException,InterruptedException{
return 0;
}
+
+ @Override
+ public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
+ org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException("Not supported");
+ }
}