You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/07/13 22:38:46 UTC
svn commit: r1361385 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanag...
Author: bobby
Date: Fri Jul 13 20:38:46 2012
New Revision: 1361385
URL: http://svn.apache.org/viewvc?rev=1361385&view=rev
Log:
svn merge -c 1298978 FIXES:MAPREDUCE-3348. Fixed a bug in MR client to redirect to JobHistoryServer correctly when RM forgets the app. Contributed by Devaraj K.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1361385&r1=1361384&r2=1361385&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Jul 13 20:38:46 2012
@@ -319,6 +319,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3940. ContainerTokens should have an expiry interval. (Siddharth
Seth and Vinod Kumar Vavilapalli via vinodkv)
+ MAPREDUCE-3348. Fixed a bug in MR client to redirect to JobHistoryServer
+ correctly when RM forgets the app. (Devaraj K via vinodkv)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1361385&r1=1361384&r2=1361385&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Fri Jul 13 20:38:46 2012
@@ -149,7 +149,7 @@ public class ClientServiceDelegate {
|| YarnApplicationState.RUNNING == application
.getYarnApplicationState()) {
if (application == null) {
- LOG.debug("Could not get Job info from RM for job " + jobId
+ LOG.info("Could not get Job info from RM for job " + jobId
+ ". Redirecting to job history server.");
return checkAndGetHSProxy(null, JobState.NEW);
}
@@ -214,7 +214,7 @@ public class ClientServiceDelegate {
}
application = rm.getApplicationReport(appId);
if (application == null) {
- LOG.debug("Could not get Job info from RM for job " + jobId
+ LOG.info("Could not get Job info from RM for job " + jobId
+ ". Redirecting to job history server.");
return checkAndGetHSProxy(null, JobState.RUNNING);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1361385&r1=1361384&r2=1361385&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Jul 13 20:38:46 2012
@@ -196,6 +196,10 @@ public class ClientRMService extends Abs
return response;
}
+ /**
+ * It gives response which includes application report if the application
+ * present otherwise gives response with application report as null.
+ */
@Override
public GetApplicationReportResponse getApplicationReport(
GetApplicationReportRequest request) throws YarnRemoteException {
@@ -211,8 +215,10 @@ public class ClientRMService extends Abs
RMApp application = this.rmContext.getRMApps().get(applicationId);
if (application == null) {
- throw RPCUtil.getRemoteException("Trying to get information for an "
- + "absent application " + applicationId);
+ // If the RM doesn't have the application, provide the response with
+ // application report as null and let the clients to handle.
+ return recordFactory
+ .newRecordInstance(GetApplicationReportResponse.class);
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1361385&r1=1361384&r2=1361385&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Fri Jul 13 20:38:46 2012
@@ -18,8 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.net.InetSocketAddress;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
import junit.framework.Assert;
@@ -27,12 +31,20 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
+
public class TestClientRMService {
private static final Log LOG = LogFactory.getLog(TestClientRMService.class);
@@ -79,4 +91,22 @@ public class TestClientRMService {
Assert.assertFalse("Node is expected to be unhealthy!", nodeReports.get(0)
.getNodeHealthStatus().getIsNodeHealthy());
}
+
+ @Test
+ public void testGetApplicationReport() throws YarnRemoteException {
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getRMApps()).thenReturn(
+ new ConcurrentHashMap<ApplicationId, RMApp>());
+ ClientRMService rmService = new ClientRMService(rmContext, null, null,
+ null, null);
+ RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ GetApplicationReportRequest request = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ request.setApplicationId(recordFactory
+ .newRecordInstance(ApplicationId.class));
+ GetApplicationReportResponse applicationReport = rmService
+ .getApplicationReport(request);
+ Assert.assertNull("It should return null as application report for absent application.",
+ applicationReport.getApplicationReport());
+ }
}