You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/06/18 06:03:44 UTC
svn commit: r1494018 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src:
main/java/org/apache/hadoop/mapred/ test/java/org/apache/hadoop/mapred/
test/java/org/apache/hadoop/mapre...
Author: vinodkv
Date: Tue Jun 18 04:03:43 2013
New Revision: 1494018
URL: http://svn.apache.org/r1494018
Log:
YARN-834. Fixed annotations for yarn-client module, reorganized packages and clearly differentiated *Async apis. Contributed by Arun C Murthy and Zhijie Shen.
svn merge --ignore-ancestry -c 1494017 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1494018&r1=1494017&r2=1494018&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Jun 18 04:03:43 2013
@@ -20,9 +20,12 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -40,34 +43,85 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ProtoUtils;
-public class ResourceMgrDelegate extends YarnClientImpl {
+import com.google.common.annotations.VisibleForTesting;
+
+public class ResourceMgrDelegate extends YarnClient {
private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
private YarnConfiguration conf;
private GetNewApplicationResponse application;
private ApplicationId applicationId;
+ @Private
+ @VisibleForTesting
+ protected YarnClient client;
+ private InetSocketAddress rmAddress;
/**
- * Delegate responsible for communicating with the Resource Manager's {@link ApplicationClientProtocol}.
+ * Delegate responsible for communicating with the Resource Manager's
+ * {@link ApplicationClientProtocol}.
* @param conf the configuration object.
*/
public ResourceMgrDelegate(YarnConfiguration conf) {
- super();
+ this(conf, null);
+ }
+
+ /**
+ * Delegate responsible for communicating with the Resource Manager's
+ * {@link ApplicationClientProtocol}.
+ * @param conf the configuration object.
+ * @param rmAddress the address of the Resource Manager
+ */
+ public ResourceMgrDelegate(YarnConfiguration conf,
+ InetSocketAddress rmAddress) {
+ super(ResourceMgrDelegate.class.getName());
this.conf = conf;
+ this.rmAddress = rmAddress;
+ if (rmAddress == null) {
+ client = YarnClient.createYarnClient();
+ } else {
+ client = YarnClient.createYarnClient(rmAddress);
+ }
init(conf);
start();
}
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ if (rmAddress == null) {
+ this.rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+ client.init(conf);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ client.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ client.stop();
+ super.serviceStop();
+ }
+
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
try {
- return TypeConverter.fromYarnNodes(super.getNodeReports());
+ return TypeConverter.fromYarnNodes(client.getNodeReports());
} catch (YarnException e) {
throw new IOException(e);
}
@@ -75,7 +129,7 @@ public class ResourceMgrDelegate extends
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
try {
- return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf);
+ return TypeConverter.fromYarnApps(client.getApplicationList(), this.conf);
} catch (YarnException e) {
throw new IOException(e);
}
@@ -91,7 +145,7 @@ public class ResourceMgrDelegate extends
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
try {
- YarnClusterMetrics metrics = super.getYarnClusterMetrics();
+ YarnClusterMetrics metrics = client.getYarnClusterMetrics();
ClusterMetrics oldMetrics =
new ClusterMetrics(1, 1, 1, 1, 1, 1,
metrics.getNumNodeManagers() * 10,
@@ -112,7 +166,7 @@ public class ResourceMgrDelegate extends
InterruptedException {
try {
return ProtoUtils.convertFromProtoFormat(
- super.getRMDelegationToken(renewer), rmAddress);
+ client.getRMDelegationToken(renewer), rmAddress);
} catch (YarnException e) {
throw new IOException(e);
}
@@ -124,7 +178,7 @@ public class ResourceMgrDelegate extends
public JobID getNewJobID() throws IOException, InterruptedException {
try {
- this.application = super.getNewApplication();
+ this.application = client.getNewApplication();
this.applicationId = this.application.getApplicationId();
return TypeConverter.fromYarn(applicationId);
} catch (YarnException e) {
@@ -136,7 +190,7 @@ public class ResourceMgrDelegate extends
InterruptedException {
try {
org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
- super.getQueueInfo(queueName);
+ client.getQueueInfo(queueName);
return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo,
conf);
} catch (YarnException e) {
@@ -147,7 +201,7 @@ public class ResourceMgrDelegate extends
public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException,
InterruptedException {
try {
- return TypeConverter.fromYarnQueueUserAclsInfo(super
+ return TypeConverter.fromYarnQueueUserAclsInfo(client
.getQueueAclsInfo());
} catch (YarnException e) {
throw new IOException(e);
@@ -156,7 +210,7 @@ public class ResourceMgrDelegate extends
public QueueInfo[] getQueues() throws IOException, InterruptedException {
try {
- return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf);
+ return TypeConverter.fromYarnQueueInfo(client.getAllQueues(), this.conf);
} catch (YarnException e) {
throw new IOException(e);
}
@@ -164,7 +218,7 @@ public class ResourceMgrDelegate extends
public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
try {
- return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(),
+ return TypeConverter.fromYarnQueueInfo(client.getRootQueueInfos(),
this.conf);
} catch (YarnException e) {
throw new IOException(e);
@@ -174,7 +228,7 @@ public class ResourceMgrDelegate extends
public QueueInfo[] getChildQueues(String parent) throws IOException,
InterruptedException {
try {
- return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent),
+ return TypeConverter.fromYarnQueueInfo(client.getChildQueueInfos(parent),
this.conf);
} catch (YarnException e) {
throw new IOException(e);
@@ -216,4 +270,82 @@ public class ResourceMgrDelegate extends
public ApplicationId getApplicationId() {
return applicationId;
}
+
+ @Override
+ public GetNewApplicationResponse getNewApplication() throws YarnException,
+ IOException {
+ return client.getNewApplication();
+ }
+
+ @Override
+ public ApplicationId
+ submitApplication(ApplicationSubmissionContext appContext)
+ throws YarnException, IOException {
+ return client.submitApplication(appContext);
+ }
+
+ @Override
+ public void killApplication(ApplicationId applicationId)
+ throws YarnException, IOException {
+ client.killApplication(applicationId);
+ }
+
+ @Override
+ public ApplicationReport getApplicationReport(ApplicationId appId)
+ throws YarnException, IOException {
+ return client.getApplicationReport(appId);
+ }
+
+ @Override
+ public List<ApplicationReport> getApplicationList() throws YarnException,
+ IOException {
+ return client.getApplicationList();
+ }
+
+ @Override
+ public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
+ IOException {
+ return client.getYarnClusterMetrics();
+ }
+
+ @Override
+ public List<NodeReport> getNodeReports() throws YarnException, IOException {
+ return client.getNodeReports();
+ }
+
+ @Override
+ public org.apache.hadoop.yarn.api.records.Token getRMDelegationToken(
+ Text renewer) throws YarnException, IOException {
+ return client.getRMDelegationToken(renewer);
+ }
+
+ @Override
+ public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(
+ String queueName) throws YarnException, IOException {
+ return client.getQueueInfo(queueName);
+ }
+
+ @Override
+ public List<org.apache.hadoop.yarn.api.records.QueueInfo> getAllQueues()
+ throws YarnException, IOException {
+ return client.getAllQueues();
+ }
+
+ @Override
+ public List<org.apache.hadoop.yarn.api.records.QueueInfo> getRootQueueInfos()
+ throws YarnException, IOException {
+ return client.getRootQueueInfos();
+ }
+
+ @Override
+ public List<org.apache.hadoop.yarn.api.records.QueueInfo> getChildQueueInfos(
+ String parent) throws YarnException, IOException {
+ return client.getChildQueueInfos(parent);
+ }
+
+ @Override
+ public List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException,
+ IOException {
+ return client.getQueueAclsInfo();
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java?rev=1494018&r1=1494017&r2=1494018&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java Tue Jun 18 04:03:43 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
@@ -67,8 +68,9 @@ public class TestResourceMgrDelegate {
ResourceMgrDelegate delegate = new ResourceMgrDelegate(
new YarnConfiguration()) {
@Override
- protected void serviceStart() {
- this.rmClient = applicationsManager;
+ protected void serviceStart() throws Exception {
+ Assert.assertTrue(this.client instanceof YarnClientImpl);
+ ((YarnClientImpl) this.client).setRMClient(applicationsManager);
}
};
delegate.getRootQueues();
@@ -110,8 +112,9 @@ public class TestResourceMgrDelegate {
ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
new YarnConfiguration()) {
@Override
- protected void serviceStart() {
- this.rmClient = applicationsManager;
+ protected void serviceStart() throws Exception {
+ Assert.assertTrue(this.client instanceof YarnClientImpl);
+ ((YarnClientImpl) this.client).setRMClient(applicationsManager);
}
};
JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?rev=1494018&r1=1494017&r2=1494018&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java Tue Jun 18 04:03:43 2013
@@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -200,8 +201,9 @@ public class TestYARNRunner extends Test
final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class);
ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
@Override
- protected void serviceStart() {
- this.rmClient = clientRMProtocol;
+ protected void serviceStart() throws Exception {
+ assertTrue(this.client instanceof YarnClientImpl);
+ ((YarnClientImpl) this.client).setRMClient(clientRMProtocol);
}
};
/* make sure kill calls finish application master */
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java?rev=1494018&r1=1494017&r2=1494018&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java Tue Jun 18 04:03:43 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -110,8 +111,9 @@ public class TestYarnClientProtocolProvi
ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate(
new YarnConfiguration(conf)) {
@Override
- protected void serviceStart() {
- this.rmClient = cRMProtocol;
+ protected void serviceStart() throws Exception {
+ assertTrue(this.client instanceof YarnClientImpl);
+ ((YarnClientImpl) this.client).setRMClient(cRMProtocol);
}
};
yrunner.setResourceMgrDelegate(rmgrDelegate);