You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/05/13 05:34:26 UTC
svn commit: r1481695 [2/2] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/...
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1481695&r1=1481694&r2=1481695&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Mon May 13 03:34:25 2013
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -103,7 +102,7 @@ public class TestClientServiceDelegate {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
- RPCUtil.getRemoteException("Job ID doesnot Exist"));
+ new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
@@ -199,8 +198,7 @@ public class TestClientServiceDelegate {
}
@Test
- public void testReconnectOnAMRestart() throws IOException,
- YarnRemoteException {
+ public void testReconnectOnAMRestart() throws IOException {
//test not applicable when AM not reachable
//as instantiateAMProxy is not called at all
if(!isAMReachableFromClient) {
@@ -212,11 +210,15 @@ public class TestClientServiceDelegate {
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
- when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
- getRunningApplicationReport("am1", 78)).thenReturn(
- getRunningApplicationReport(null, 0)).thenReturn(
- getRunningApplicationReport(null, 0)).thenReturn(
- getRunningApplicationReport("am2", 90));
+ try {
+ when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
+ getRunningApplicationReport("am1", 78)).thenReturn(
+ getRunningApplicationReport(null, 0)).thenReturn(
+ getRunningApplicationReport(null, 0)).thenReturn(
+ getRunningApplicationReport("am2", 90));
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
when(jobReportResponse1.getJobReport()).thenReturn(
@@ -267,7 +269,7 @@ public class TestClientServiceDelegate {
}
@Test
- public void testAMAccessDisabled() throws IOException, YarnRemoteException {
+ public void testAMAccessDisabled() throws IOException {
//test only applicable when AM not reachable
if(isAMReachableFromClient) {
return;
@@ -278,11 +280,15 @@ public class TestClientServiceDelegate {
getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
- when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
- getRunningApplicationReport("am1", 78)).thenReturn(
+ try {
+ when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
getRunningApplicationReport("am1", 78)).thenReturn(
getRunningApplicationReport("am1", 78)).thenReturn(
- getFinishedApplicationReport());
+ getRunningApplicationReport("am1", 78)).thenReturn(
+ getFinishedApplicationReport());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
historyServerProxy, rmDelegate));
@@ -319,8 +325,7 @@ public class TestClientServiceDelegate {
}
@Test
- public void testRMDownForJobStatusBeforeGetAMReport() throws IOException,
- YarnRemoteException {
+ public void testRMDownForJobStatusBeforeGetAMReport() throws IOException {
Configuration conf = new YarnConfiguration();
testRMDownForJobStatusBeforeGetAMReport(conf,
MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
@@ -328,7 +333,7 @@ public class TestClientServiceDelegate {
@Test
public void testRMDownForJobStatusBeforeGetAMReportWithRetryTimes()
- throws IOException, YarnRemoteException {
+ throws IOException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 2);
testRMDownForJobStatusBeforeGetAMReport(conf, conf.getInt(
@@ -338,7 +343,7 @@ public class TestClientServiceDelegate {
@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport()
- throws IOException, YarnRemoteException {
+ throws IOException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
@@ -349,42 +354,52 @@ public class TestClientServiceDelegate {
when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(getJobReportResponse());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
- when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
- new java.lang.reflect.UndeclaredThrowableException(new IOException(
- "Connection refuced1"))).thenThrow(
- new java.lang.reflect.UndeclaredThrowableException(new IOException(
- "Connection refuced2"))).thenReturn(getFinishedApplicationReport());
- ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
- conf, rmDelegate, oldJobId, historyServerProxy);
- JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
- verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
- Assert.assertNotNull(jobStatus);
+ try {
+ when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
+ new java.lang.reflect.UndeclaredThrowableException(new IOException(
+ "Connection refuced1"))).thenThrow(
+ new java.lang.reflect.UndeclaredThrowableException(new IOException(
+ "Connection refuced2")))
+ .thenReturn(getFinishedApplicationReport());
+ ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
+ conf, rmDelegate, oldJobId, historyServerProxy);
+ JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
+ verify(rmDelegate, times(3)).getApplicationReport(
+ any(ApplicationId.class));
+ Assert.assertNotNull(jobStatus);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf,
- int noOfRetries) throws YarnRemoteException, IOException {
+ int noOfRetries) throws IOException {
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
!isAMReachableFromClient);
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
- when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
- new java.lang.reflect.UndeclaredThrowableException(new IOException(
- "Connection refuced1"))).thenThrow(
- new java.lang.reflect.UndeclaredThrowableException(new IOException(
- "Connection refuced2"))).thenThrow(
- new java.lang.reflect.UndeclaredThrowableException(new IOException(
- "Connection refuced3")));
- ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
- conf, rmDelegate, oldJobId, historyServerProxy);
try {
- clientServiceDelegate.getJobStatus(oldJobId);
- Assert.fail("It should throw exception after retries");
- } catch (IOException e) {
- System.out.println("fail to get job status,and e=" + e.toString());
+ when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
+ new java.lang.reflect.UndeclaredThrowableException(new IOException(
+ "Connection refuced1"))).thenThrow(
+ new java.lang.reflect.UndeclaredThrowableException(new IOException(
+ "Connection refuced2"))).thenThrow(
+ new java.lang.reflect.UndeclaredThrowableException(new IOException(
+ "Connection refuced3")));
+ ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
+ conf, rmDelegate, oldJobId, historyServerProxy);
+ try {
+ clientServiceDelegate.getJobStatus(oldJobId);
+ Assert.fail("It should throw exception after retries");
+ } catch (IOException e) {
+ System.out.println("fail to get job status,and e=" + e.toString());
+ }
+ verify(rmDelegate, times(noOfRetries)).getApplicationReport(
+ any(ApplicationId.class));
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
}
- verify(rmDelegate, times(noOfRetries)).getApplicationReport(
- any(ApplicationId.class));
}
private GetJobReportRequest getJobReportRequest() {
@@ -429,10 +444,13 @@ public class TestClientServiceDelegate {
"N/A", 0.0f);
}
- private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException,
- IOException {
+ private ResourceMgrDelegate getRMDelegate() throws IOException {
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
- when(rm.getApplicationReport(jobId.getAppId())).thenReturn(null);
+ try {
+ when(rm.getApplicationReport(jobId.getAppId())).thenReturn(null);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
return rm;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java?rev=1481695&r1=1481694&r2=1481695&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java Mon May 13 03:34:25 2013
@@ -48,19 +48,21 @@ public class TestResourceMgrDelegate {
/**
* Tests that getRootQueues makes a request for the (recursive) child queues
- * @throws YarnRemoteException
* @throws IOException
*/
@Test
- public void testGetRootQueues() throws IOException, InterruptedException,
- YarnRemoteException {
+ public void testGetRootQueues() throws IOException, InterruptedException {
final ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class);
GetQueueInfoResponse response = Mockito.mock(GetQueueInfoResponse.class);
org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
Mockito.when(response.getQueueInfo()).thenReturn(queueInfo);
- Mockito.when(applicationsManager.getQueueInfo(Mockito.any(
- GetQueueInfoRequest.class))).thenReturn(response);
+ try {
+ Mockito.when(applicationsManager.getQueueInfo(Mockito.any(
+ GetQueueInfoRequest.class))).thenReturn(response);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
ResourceMgrDelegate delegate = new ResourceMgrDelegate(
new YarnConfiguration()) {
@@ -73,8 +75,12 @@ public class TestResourceMgrDelegate {
ArgumentCaptor<GetQueueInfoRequest> argument =
ArgumentCaptor.forClass(GetQueueInfoRequest.class);
- Mockito.verify(applicationsManager).getQueueInfo(
- argument.capture());
+ try {
+ Mockito.verify(applicationsManager).getQueueInfo(
+ argument.capture());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
Assert.assertTrue("Children of root queue not requested",
argument.getValue().getIncludeChildQueues());
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1481695&r1=1481694&r2=1481695&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java Mon May 13 03:34:25 2013
@@ -175,6 +175,7 @@ public class TestMRJobClient extends Clu
runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
fail(" this task should be killed");
} catch (IOException e) {
+ System.out.println(e);
// task completed
assertTrue(e.getMessage().contains("_0001_m_000000_1"));
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1481695&r1=1481694&r2=1481695&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java Mon May 13 03:34:25 2013
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
@@ -48,7 +47,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.ProtoUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -62,8 +60,7 @@ public class TestJHSSecurity {
private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class);
@Test
- public void testDelegationToken() throws IOException, InterruptedException,
- YarnRemoteException {
+ public void testDelegationToken() throws IOException, InterruptedException {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
@@ -124,7 +121,7 @@ public class TestJHSSecurity {
jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
try {
clientUsingDT.getJobReport(jobReportRequest);
- } catch (YarnRemoteException e) {
+ } catch (IOException e) {
Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
}
@@ -147,7 +144,7 @@ public class TestJHSSecurity {
// Valid token because of renewal.
try {
clientUsingDT.getJobReport(jobReportRequest);
- } catch (UndeclaredThrowableException e) {
+ } catch (IOException e) {
Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
}
@@ -161,7 +158,7 @@ public class TestJHSSecurity {
try {
clientUsingDT.getJobReport(jobReportRequest);
fail("Should not have succeeded with an expired token");
- } catch (UndeclaredThrowableException e) {
+ } catch (IOException e) {
assertTrue(e.getCause().getMessage().contains("is expired"));
}
@@ -183,7 +180,7 @@ public class TestJHSSecurity {
try {
clientUsingDT.getJobReport(jobReportRequest);
- } catch (UndeclaredThrowableException e) {
+ } catch (IOException e) {
fail("Unexpected exception" + e);
}
cancelDelegationToken(loggedInUser, hsService, token);
@@ -200,7 +197,7 @@ public class TestJHSSecurity {
try {
clientUsingDT.getJobReport(jobReportRequest);
fail("Should not have succeeded with a cancelled delegation token");
- } catch (UndeclaredThrowableException e) {
+ } catch (IOException e) {
}
@@ -219,7 +216,7 @@ public class TestJHSSecurity {
DelegationToken token = loggedInUser
.doAs(new PrivilegedExceptionAction<DelegationToken>() {
@Override
- public DelegationToken run() throws YarnRemoteException {
+ public DelegationToken run() throws IOException {
GetDelegationTokenRequest request = Records
.newRecord(GetDelegationTokenRequest.class);
request.setRenewer(renewerString);
@@ -236,7 +233,7 @@ public class TestJHSSecurity {
long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction<Long>() {
@Override
- public Long run() throws YarnRemoteException {
+ public Long run() throws IOException {
RenewDelegationTokenRequest request = Records
.newRecord(RenewDelegationTokenRequest.class);
request.setDelegationToken(dToken);
@@ -252,7 +249,7 @@ public class TestJHSSecurity {
loggedInUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws YarnRemoteException {
+ public Void run() throws IOException {
CancelDelegationTokenRequest request = Records
.newRecord(CancelDelegationTokenRequest.class);
request.setDelegationToken(dToken);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1481695&r1=1481694&r2=1481695&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Mon May 13 03:34:25 2013
@@ -46,7 +46,6 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -115,7 +114,7 @@ public class TestMRJobsWithHistoryServic
@Test (timeout = 30000)
public void testJobHistoryData() throws IOException, InterruptedException,
- AvroRemoteException, ClassNotFoundException, YarnRemoteException {
+ AvroRemoteException, ClassNotFoundException {
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ " not found. Not running test.");