You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2013/03/24 16:56:22 UTC
svn commit: r1460410 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoco...
Author: suresh
Date: Sun Mar 24 15:56:18 2013
New Revision: 1460410
URL: http://svn.apache.org/r1460410
Log:
Merging trunk to branch HDFS-2802
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientAsync.java
- copied unchanged from r1460389, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientAsync.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java
- copied unchanged from r1460389, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueUserAclsInfoResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Priority.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Sun Mar 24 15:56:18 2013
@@ -18,6 +18,13 @@ Trunk - Unreleased
BUG FIXES
+ YARN-488. TestContainerManagerSecurity fails on Windows. (Chris Nauroth
+ via hitesh)
+
+ YARN-490. TestDistributedShell fails on Windows. (Chris Nauroth via hitesh)
+
+ YARN-491. TestContainerLogsPage fails on Windows. (Chris Nauroth via hitesh)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS
YARN-158. Yarn creating package-info.java must not depend on sh.
@@ -52,6 +59,9 @@ Release 2.0.5-beta - UNRELEASED
INCOMPATIBLE CHANGES
+ YARN-396. Rationalize AllocateResponse in RM Scheduler API. (Zhijie Shen
+ via hitesh)
+
NEW FEATURES
IMPROVEMENTS
@@ -72,6 +82,15 @@ Release 2.0.5-beta - UNRELEASED
YARN-237. Refreshing the RM page forgets how many rows I had in my
Datatables (jian he via bobby)
+ YARN-481. Add AM Host and RPC Port to ApplicationCLI Status Output
+ (Chris Riccomini via bikas)
+
+ YARN-297. Improve hashCode implementations for PB records. (Xuan Gong via
+ hitesh)
+
+ YARN-417. Create AMRMClient wrapper that provides asynchronous callbacks.
+ (Sandy Ryza via bikas)
+
OPTIMIZATIONS
BUG FIXES
@@ -100,6 +119,12 @@ Release 2.0.5-beta - UNRELEASED
YARN-196. Nodemanager should be more robust in handling connection failure
to ResourceManager when a cluster is started (Xuan Gong via hitesh)
+ YARN-485. TestProcfsProcessTree#testProcessTree() doesn't wait long enough
+ for the process to die. (kkambatl via tucu)
+
+ YARN-470. Support a way to disable resource monitoring on the NodeManager.
+ (Siddharth Seth via hitesh)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
@@ -408,6 +433,9 @@ Release 0.23.7 - UNRELEASED
YARN-468. coverage fix for org.apache.hadoop.yarn.server.webproxy.amfilter
(Aleksey Gorshkov via bobby)
+ YARN-200. yarn log does not output all needed information, and is in a
+ binary format (Ravi Prakash via jlowe)
+
OPTIMIZATIONS
YARN-357. App submission should not be synchronized (daryn)
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java Sun Mar 24 15:56:18 2013
@@ -18,19 +18,23 @@
package org.apache.hadoop.yarn.api.protocolrecords;
+import java.util.List;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
/**
* <p>The response sent by the <code>ResourceManager</code> the
* <code>ApplicationMaster</code> during resource negotiation.</p>
*
- * <p>The response, via {@link AMResponse}, includes:
+ * <p>The response, includes:
* <ul>
* <li>Response ID to track duplicate responses.</li>
* <li>
@@ -42,6 +46,8 @@ import org.apache.hadoop.yarn.api.record
* The available headroom for resources in the cluster for the
* application.
* </li>
+ * <li>A list of nodes whose status has been updated.</li>
+ * <li>The number of available nodes in a cluster.</li>
* </ul>
* </p>
*
@@ -51,18 +57,90 @@ import org.apache.hadoop.yarn.api.record
@Stable
public interface AllocateResponse {
/**
- * Get the {@link AMResponse} sent by the <code>ResourceManager</code>.
- * @return <code>AMResponse</code> sent by the <code>ResourceManager</code>
+ * Should the <code>ApplicationMaster</code> reboot for being horribly
+ * out-of-sync with the <code>ResourceManager</code> as deigned by
+ * {@link #getResponseId()}?
+ *
+ * @return <code>true</code> if the <code>ApplicationMaster</code> should
+ * reboot, <code>false</code> otherwise
*/
@Public
@Stable
- public abstract AMResponse getAMResponse();
+ public boolean getReboot();
@Private
@Unstable
- public abstract void setAMResponse(AMResponse amResponse);
-
-
+ public void setReboot(boolean reboot);
+
+ /**
+ * Get the <em>last response id</em>.
+ * @return <em>last response id</em>
+ */
+ @Public
+ @Stable
+ public int getResponseId();
+
+ @Private
+ @Unstable
+ public void setResponseId(int responseId);
+
+ /**
+ * Get the list of <em>newly allocated</em> <code>Container</code> by the
+ * <code>ResourceManager</code>.
+ * @return list of <em>newly allocated</em> <code>Container</code>
+ */
+ @Public
+ @Stable
+ public List<Container> getAllocatedContainers();
+
+ /**
+ * Set the list of <em>newly allocated</em> <code>Container</code> by the
+ * <code>ResourceManager</code>.
+ * @param containers list of <em>newly allocated</em> <code>Container</code>
+ */
+ @Public
+ @Stable
+ public void setAllocatedContainers(List<Container> containers);
+
+ /**
+ * Get the <em>available headroom</em> for resources in the cluster for the
+ * application.
+ * @return limit of available headroom for resources in the cluster for the
+ * application
+ */
+ @Public
+ @Stable
+ public Resource getAvailableResources();
+
+ @Private
+ @Unstable
+ public void setAvailableResources(Resource limit);
+
+ /**
+ * Get the list of <em>completed containers' statuses</em>.
+ * @return the list of <em>completed containers' statuses</em>
+ */
+ @Public
+ @Stable
+ public List<ContainerStatus> getCompletedContainersStatuses();
+
+ @Private
+ @Unstable
+ public void setCompletedContainersStatuses(List<ContainerStatus> containers);
+
+ /**
+ * Get the list of <em>updated <code>NodeReport</code>s</em>. Updates could
+ * be changes in health, availability etc of the nodes.
+ * @return The delta of updated nodes since the last response
+ */
+ @Public
+ @Unstable
+ public List<NodeReport> getUpdatedNodes();
+
+ @Private
+ @Unstable
+ public void setUpdatedNodes(final List<NodeReport> updatedNodes);
+
/**
* Get the number of hosts available on the cluster.
* @return the available host count.
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Sun Mar 24 15:56:18 2013
@@ -19,11 +19,24 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.ProtoBase;
-import org.apache.hadoop.yarn.api.records.impl.pb.AMResponsePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProto;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProtoOrBuilder;
@@ -35,7 +48,12 @@ public class AllocateResponsePBImpl exte
AllocateResponseProto.Builder builder = null;
boolean viaProto = false;
- private AMResponse amResponse;
+ Resource limit;
+
+ private List<Container> allocatedContainers = null;
+ private List<ContainerStatus> completedContainersStatuses = null;
+
+ private List<NodeReport> updatedNodes = null;
public AllocateResponsePBImpl() {
@@ -47,20 +65,38 @@ public class AllocateResponsePBImpl exte
viaProto = true;
}
- public AllocateResponseProto getProto() {
+ public synchronized AllocateResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void mergeLocalToBuilder() {
- if (this.amResponse != null) {
- builder.setAMResponse(convertToProtoFormat(this.amResponse));
+ private synchronized void mergeLocalToBuilder() {
+ if (this.allocatedContainers != null) {
+ builder.clearAllocatedContainers();
+ Iterable<ContainerProto> iterable =
+ getProtoIterable(this.allocatedContainers);
+ builder.addAllAllocatedContainers(iterable);
+ }
+ if (this.completedContainersStatuses != null) {
+ builder.clearCompletedContainerStatuses();
+ Iterable<ContainerStatusProto> iterable =
+ getContainerStatusProtoIterable(this.completedContainersStatuses);
+ builder.addAllCompletedContainerStatuses(iterable);
+ }
+ if (this.updatedNodes != null) {
+ builder.clearUpdatedNodes();
+ Iterable<NodeReportProto> iterable =
+ getNodeReportProtoIterable(this.updatedNodes);
+ builder.addAllUpdatedNodes(iterable);
+ }
+ if (this.limit != null) {
+ builder.setLimit(convertToProtoFormat(this.limit));
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
@@ -68,53 +104,293 @@ public class AllocateResponsePBImpl exte
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = AllocateResponseProto.newBuilder(proto);
}
viaProto = false;
}
-
@Override
- public AMResponse getAMResponse() {
+ public synchronized boolean getReboot() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.amResponse != null) {
- return this.amResponse;
+ return (p.getReboot());
+ }
+
+ @Override
+ public synchronized void setReboot(boolean reboot) {
+ maybeInitBuilder();
+ builder.setReboot((reboot));
+ }
+
+ @Override
+ public synchronized int getResponseId() {
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getResponseId());
+ }
+
+ @Override
+ public synchronized void setResponseId(int responseId) {
+ maybeInitBuilder();
+ builder.setResponseId((responseId));
+ }
+
+ @Override
+ public synchronized Resource getAvailableResources() {
+ if (this.limit != null) {
+ return this.limit;
}
- if (!p.hasAMResponse()) {
+
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasLimit()) {
return null;
}
- this.amResponse= convertFromProtoFormat(p.getAMResponse());
- return this.amResponse;
+ this.limit = convertFromProtoFormat(p.getLimit());
+ return this.limit;
}
@Override
- public void setAMResponse(AMResponse aMResponse) {
+ public synchronized void setAvailableResources(Resource limit) {
maybeInitBuilder();
- if (aMResponse == null)
- builder.clearAMResponse();
- this.amResponse = aMResponse;
+ if (limit == null)
+ builder.clearLimit();
+ this.limit = limit;
}
-
+
+ @Override
+ public synchronized List<NodeReport> getUpdatedNodes() {
+ initLocalNewNodeReportList();
+ return this.updatedNodes;
+ }
+ @Override
+ public synchronized void setUpdatedNodes(
+ final List<NodeReport> updatedNodes) {
+ if (updatedNodes == null) {
+ this.updatedNodes.clear();
+ return;
+ }
+ this.updatedNodes = new ArrayList<NodeReport>(updatedNodes.size());
+ this.updatedNodes.addAll(updatedNodes);
+ }
+
+ @Override
+ public synchronized List<Container> getAllocatedContainers() {
+ initLocalNewContainerList();
+ return this.allocatedContainers;
+ }
+
@Override
- public int getNumClusterNodes() {
+ public synchronized void setAllocatedContainers(
+ final List<Container> containers) {
+ if (containers == null)
+ return;
+ // this looks like a bug because it results in append and not set
+ initLocalNewContainerList();
+ allocatedContainers.addAll(containers);
+ }
+
+ //// Finished containers
+ @Override
+ public synchronized List<ContainerStatus> getCompletedContainersStatuses() {
+ initLocalFinishedContainerList();
+ return this.completedContainersStatuses;
+ }
+
+ @Override
+ public synchronized void setCompletedContainersStatuses(
+ final List<ContainerStatus> containers) {
+ if (containers == null)
+ return;
+ initLocalFinishedContainerList();
+ completedContainersStatuses.addAll(containers);
+ }
+
+ @Override
+ public synchronized int getNumClusterNodes() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
return p.getNumClusterNodes();
}
-
+
@Override
- public void setNumClusterNodes(int numNodes) {
+ public synchronized void setNumClusterNodes(int numNodes) {
maybeInitBuilder();
builder.setNumClusterNodes(numNodes);
}
+ // Once this is called. updatedNodes will never be null - until a getProto is
+ // called.
+ private synchronized void initLocalNewNodeReportList() {
+ if (this.updatedNodes != null) {
+ return;
+ }
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<NodeReportProto> list = p.getUpdatedNodesList();
+ updatedNodes = new ArrayList<NodeReport>(list.size());
+
+ for (NodeReportProto n : list) {
+ updatedNodes.add(convertFromProtoFormat(n));
+ }
+ }
+
+ // Once this is called. containerList will never be null - until a getProto
+ // is called.
+ private synchronized void initLocalNewContainerList() {
+ if (this.allocatedContainers != null) {
+ return;
+ }
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerProto> list = p.getAllocatedContainersList();
+ allocatedContainers = new ArrayList<Container>();
+
+ for (ContainerProto c : list) {
+ allocatedContainers.add(convertFromProtoFormat(c));
+ }
+ }
+
+ private synchronized Iterable<ContainerProto> getProtoIterable(
+ final List<Container> newContainersList) {
+ maybeInitBuilder();
+ return new Iterable<ContainerProto>() {
+ @Override
+ public synchronized Iterator<ContainerProto> iterator() {
+ return new Iterator<ContainerProto>() {
+
+ Iterator<Container> iter = newContainersList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized ContainerProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ }
+
+ private synchronized Iterable<ContainerStatusProto>
+ getContainerStatusProtoIterable(
+ final List<ContainerStatus> newContainersList) {
+ maybeInitBuilder();
+ return new Iterable<ContainerStatusProto>() {
+ @Override
+ public synchronized Iterator<ContainerStatusProto> iterator() {
+ return new Iterator<ContainerStatusProto>() {
+
+ Iterator<ContainerStatus> iter = newContainersList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized ContainerStatusProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ }
- private AMResponsePBImpl convertFromProtoFormat(AMResponseProto p) {
- return new AMResponsePBImpl(p);
+ private synchronized Iterable<NodeReportProto>
+ getNodeReportProtoIterable(
+ final List<NodeReport> newNodeReportsList) {
+ maybeInitBuilder();
+ return new Iterable<NodeReportProto>() {
+ @Override
+ public synchronized Iterator<NodeReportProto> iterator() {
+ return new Iterator<NodeReportProto>() {
+
+ Iterator<NodeReport> iter = newNodeReportsList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized NodeReportProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ }
+
+ // Once this is called. containerList will never be null - until a getProto
+ // is called.
+ private synchronized void initLocalFinishedContainerList() {
+ if (this.completedContainersStatuses != null) {
+ return;
+ }
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerStatusProto> list = p.getCompletedContainerStatusesList();
+ completedContainersStatuses = new ArrayList<ContainerStatus>();
+
+ for (ContainerStatusProto c : list) {
+ completedContainersStatuses.add(convertFromProtoFormat(c));
+ }
+ }
+
+ private synchronized NodeReportPBImpl convertFromProtoFormat(
+ NodeReportProto p) {
+ return new NodeReportPBImpl(p);
+ }
+
+ private synchronized NodeReportProto convertToProtoFormat(NodeReport t) {
+ return ((NodeReportPBImpl)t).getProto();
}
- private AMResponseProto convertToProtoFormat(AMResponse t) {
- return ((AMResponsePBImpl)t).getProto();
+ private synchronized ContainerPBImpl convertFromProtoFormat(
+ ContainerProto p) {
+ return new ContainerPBImpl(p);
}
+
+ private synchronized ContainerProto convertToProtoFormat(Container t) {
+ return ((ContainerPBImpl)t).getProto();
+ }
+
+ private synchronized ContainerStatusPBImpl convertFromProtoFormat(
+ ContainerStatusProto p) {
+ return new ContainerStatusPBImpl(p);
+ }
+
+ private synchronized ContainerStatusProto convertToProtoFormat(
+ ContainerStatus t) {
+ return ((ContainerStatusPBImpl)t).getProto();
+ }
+
+ private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private synchronized ResourceProto convertToProtoFormat(Resource r) {
+ return ((ResourcePBImpl) r).getProto();
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java Sun Mar 24 15:56:18 2013
@@ -93,7 +93,8 @@ GetAllApplicationsResponse {
viaProto = false;
}
- //Once this is called. containerList will never be null - untill a getProto is called.
+ // Once this is called. containerList will never be null - until a getProto
+ // is called.
private void initLocalApplicationsList() {
if (this.applicationList != null) {
return;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java Sun Mar 24 15:56:18 2013
@@ -92,7 +92,8 @@ public class GetClusterNodesResponsePBIm
viaProto = false;
}
- //Once this is called. containerList will never be null - untill a getProto is called.
+ // Once this is called. containerList will never be null - until a getProto
+ // is called.
private void initLocalNodeManagerInfosList() {
if (this.nodeManagerInfoList != null) {
return;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueUserAclsInfoResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueUserAclsInfoResponsePBImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueUserAclsInfoResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueUserAclsInfoResponsePBImpl.java Sun Mar 24 15:56:18 2013
@@ -94,7 +94,8 @@ implements GetQueueUserAclsInfoResponse
viaProto = false;
}
- //Once this is called. containerList will never be null - untill a getProto is called.
+ // Once this is called. containerList will never be null - until a getProto
+ // is called.
private void initLocalQueueUserAclsList() {
if (this.queueUserAclsInfoList != null) {
return;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java Sun Mar 24 15:56:18 2013
@@ -76,8 +76,8 @@ public abstract class ApplicationAttempt
@Override
public int hashCode() {
// Generated by eclipse.
- final int prime = 31;
- int result = 1;
+ final int prime = 347671;
+ int result = 5501;
ApplicationId appId = getApplicationId();
result = prime * result + appId.hashCode();
result = prime * result + getAttemptId();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java Sun Mar 24 15:56:18 2013
@@ -97,8 +97,8 @@ public abstract class ApplicationId impl
@Override
public int hashCode() {
// Generated by eclipse.
- final int prime = 31;
- int result = 1;
+ final int prime = 371237;
+ int result = 6521;
long clusterTimestamp = getClusterTimestamp();
result = prime * result
+ (int) (clusterTimestamp ^ (clusterTimestamp >>> 32));
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java Sun Mar 24 15:56:18 2013
@@ -86,8 +86,8 @@ public abstract class ContainerId implem
@Override
public int hashCode() {
// Generated by eclipse.
- final int prime = 31;
- int result = 1;
+ final int prime = 435569;
+ int result = 7507;
result = prime * result + getId();
result = prime * result + getApplicationAttemptId().hashCode();
return result;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java Sun Mar 24 15:56:18 2013
@@ -65,8 +65,8 @@ public abstract class NodeId implements
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
+ final int prime = 493217;
+ int result = 8501;
result = prime * result + this.getHost().hashCode();
result = prime * result + this.getPort();
return result;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Priority.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Priority.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Priority.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Priority.java Sun Mar 24 15:56:18 2013
@@ -39,8 +39,8 @@ public abstract class Priority implement
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
+ final int prime = 517861;
+ int result = 9511;
result = prime * result + getPriority();
return result;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java Sun Mar 24 15:56:18 2013
@@ -124,8 +124,8 @@ public abstract class ResourceRequest im
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
+ final int prime = 2153;
+ int result = 2459;
Resource capability = getCapability();
String hostName = getHostName();
Priority priority = getPriority();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Sun Mar 24 15:56:18 2013
@@ -207,16 +207,6 @@ message ResourceRequestProto {
optional int32 num_containers = 4;
}
-message AMResponseProto {
- optional bool reboot = 1;
- optional int32 response_id = 2;
- repeated ContainerProto allocated_containers = 3;
- repeated ContainerStatusProto completed_container_statuses = 4;
- optional ResourceProto limit = 5;
- repeated NodeReportProto updated_nodes = 6;
-}
-
-
////////////////////////////////////////////////////////////////////////
////// From client_RM_Protocol /////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Sun Mar 24 15:56:18 2013
@@ -59,8 +59,13 @@ message AllocateRequestProto {
}
message AllocateResponseProto {
- optional AMResponseProto AM_response = 1;
- optional int32 num_cluster_nodes = 2;
+ optional bool reboot = 1;
+ optional int32 response_id = 2;
+ repeated ContainerProto allocated_containers = 3;
+ repeated ContainerStatusProto completed_container_statuses = 4;
+ optional ResourceProto limit = 5;
+ repeated NodeReportProto updated_nodes = 6;
+ optional int32 num_cluster_nodes = 7;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Sun Mar 24 15:56:18 2013
@@ -53,7 +53,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -64,12 +63,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.client.AMRMClient;
import org.apache.hadoop.yarn.client.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.AMRMClientImpl;
+import org.apache.hadoop.yarn.client.AMRMClientAsync;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -148,8 +147,8 @@ public class ApplicationMaster {
private YarnRPC rpc;
// Handle to communicate with the Resource Manager
- private AMRMClient resourceManager;
-
+ private AMRMClientAsync resourceManager;
+
// Application Attempt Id ( combination of attemptId and fail count )
private ApplicationAttemptId appAttemptID;
@@ -170,8 +169,6 @@ public class ApplicationMaster {
// Priority of the request
private int requestPriority;
- // Simple flag to denote whether all works is done
- private boolean appDone = false;
// Counter for completed containers ( complete denotes successful or failed )
private AtomicInteger numCompletedContainers = new AtomicInteger();
// Allocated container count so that we know how many containers has the RM
@@ -202,6 +199,9 @@ public class ApplicationMaster {
// Hardcoded path to shell script in launch container's local env
private final String ExecShellStringPath = "ExecShellScript.sh";
+ private volatile boolean done;
+ private volatile boolean success;
+
// Launch threads
private List<Thread> launchThreads = new ArrayList<Thread>();
@@ -394,6 +394,10 @@ public class ApplicationMaster {
"container_memory", "10"));
numTotalContainers = Integer.parseInt(cliParser.getOptionValue(
"num_containers", "1"));
+ if (numTotalContainers == 0) {
+ throw new IllegalArgumentException(
+ "Cannot run distributed shell with no containers");
+ }
requestPriority = Integer.parseInt(cliParser
.getOptionValue("priority", "0"));
@@ -417,225 +421,202 @@ public class ApplicationMaster {
public boolean run() throws YarnRemoteException {
LOG.info("Starting ApplicationMaster");
- // Connect to ResourceManager
- resourceManager = new AMRMClientImpl(appAttemptID);
+ AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
+
+ resourceManager = new AMRMClientAsync(appAttemptID, 1000, allocListener);
resourceManager.init(conf);
resourceManager.start();
- try {
- // Setup local RPC Server to accept status requests directly from clients
- // TODO need to setup a protocol for client to be able to communicate to
- // the RPC server
- // TODO use the rpc port info to register with the RM for the client to
- // send requests to this app master
-
- // Register self with ResourceManager
- RegisterApplicationMasterResponse response = resourceManager
- .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
- appMasterTrackingUrl);
- // Dump out information about cluster capability as seen by the
- // resource manager
- int minMem = response.getMinimumResourceCapability().getMemory();
- int maxMem = response.getMaximumResourceCapability().getMemory();
- LOG.info("Min mem capabililty of resources in this cluster " + minMem);
- LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
-
- // A resource ask has to be atleast the minimum of the capability of the
- // cluster, the value has to be a multiple of the min value and cannot
- // exceed the max.
- // If it is not an exact multiple of min, the RM will allocate to the
- // nearest multiple of min
- if (containerMemory < minMem) {
- LOG.info("Container memory specified below min threshold of cluster."
- + " Using min value." + ", specified=" + containerMemory + ", min="
- + minMem);
- containerMemory = minMem;
- } else if (containerMemory > maxMem) {
- LOG.info("Container memory specified above max threshold of cluster."
- + " Using max value." + ", specified=" + containerMemory + ", max="
- + maxMem);
- containerMemory = maxMem;
- }
-
- // Setup heartbeat emitter
- // TODO poll RM every now and then with an empty request to let RM know
- // that we are alive
- // The heartbeat interval after which an AM is timed out by the RM is
- // defined by a config setting:
- // RM_AM_EXPIRY_INTERVAL_MS with default defined by
- // DEFAULT_RM_AM_EXPIRY_INTERVAL_MS
- // The allocate calls to the RM count as heartbeats so, for now,
- // this additional heartbeat emitter is not required.
-
- // Setup ask for containers from RM
- // Send request for containers to RM
- // Until we get our fully allocated quota, we keep on polling RM for
- // containers
- // Keep looping until all the containers are launched and shell script
- // executed on them ( regardless of success/failure).
-
- int loopCounter = -1;
-
- while (numCompletedContainers.get() < numTotalContainers && !appDone) {
- loopCounter++;
-
- // log current state
- LOG.info("Current application state: loop=" + loopCounter
- + ", appDone=" + appDone + ", total=" + numTotalContainers
- + ", requested=" + numRequestedContainers + ", completed="
- + numCompletedContainers + ", failed=" + numFailedContainers
- + ", currentAllocated=" + numAllocatedContainers);
-
- // Sleep before each loop when asking RM for containers
- // to avoid flooding RM with spurious requests when it
- // need not have any available containers
- // Sleeping for 1000 ms.
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOG.info("Sleep interrupted " + e.getMessage());
- }
+ // Setup local RPC Server to accept status requests directly from clients
+ // TODO need to setup a protocol for client to be able to communicate to
+ // the RPC server
+ // TODO use the rpc port info to register with the RM for the client to
+ // send requests to this app master
+
+ // Register self with ResourceManager
+ // This will start heartbeating to the RM
+ RegisterApplicationMasterResponse response = resourceManager
+ .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+ appMasterTrackingUrl);
+ // Dump out information about cluster capability as seen by the
+ // resource manager
+ int minMem = response.getMinimumResourceCapability().getMemory();
+ int maxMem = response.getMaximumResourceCapability().getMemory();
+ LOG.info("Min mem capabililty of resources in this cluster " + minMem);
+ LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
+
+ // A resource ask has to be atleast the minimum of the capability of the
+ // cluster, the value has to be a multiple of the min value and cannot
+ // exceed the max.
+ // If it is not an exact multiple of min, the RM will allocate to the
+ // nearest multiple of min
+ if (containerMemory < minMem) {
+ LOG.info("Container memory specified below min threshold of cluster."
+ + " Using min value." + ", specified=" + containerMemory + ", min="
+ + minMem);
+ containerMemory = minMem;
+ } else if (containerMemory > maxMem) {
+ LOG.info("Container memory specified above max threshold of cluster."
+ + " Using max value." + ", specified=" + containerMemory + ", max="
+ + maxMem);
+ containerMemory = maxMem;
+ }
- // No. of containers to request
- // For the first loop, askCount will be equal to total containers needed
- // From that point on, askCount will always be 0 as current
- // implementation does not change its ask on container failures.
- int askCount = numTotalContainers - numRequestedContainers.get();
- numRequestedContainers.addAndGet(askCount);
-
- if (askCount > 0) {
- ContainerRequest containerAsk = setupContainerAskForRM(askCount);
- resourceManager.addContainerRequest(containerAsk);
- }
- // Send the request to RM
- LOG.info("Asking RM for containers" + ", askCount=" + askCount);
- AMResponse amResp = sendContainerAskToRM();
-
- // Retrieve list of allocated containers from the response
- List<Container> allocatedContainers = amResp.getAllocatedContainers();
- LOG.info("Got response from RM for container ask, allocatedCnt="
- + allocatedContainers.size());
- numAllocatedContainers.addAndGet(allocatedContainers.size());
- for (Container allocatedContainer : allocatedContainers) {
- LOG.info("Launching shell command on a new container."
- + ", containerId=" + allocatedContainer.getId()
- + ", containerNode=" + allocatedContainer.getNodeId().getHost()
- + ":" + allocatedContainer.getNodeId().getPort()
- + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress()
- + ", containerState" + allocatedContainer.getState()
- + ", containerResourceMemory"
- + allocatedContainer.getResource().getMemory());
- // + ", containerToken"
- // +allocatedContainer.getContainerToken().getIdentifier().toString());
-
- LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(
- allocatedContainer);
- Thread launchThread = new Thread(runnableLaunchContainer);
-
- // launch and start the container on a separate thread to keep
- // the main thread unblocked
- // as all containers may not be allocated at one go.
- launchThreads.add(launchThread);
- launchThread.start();
- }
+ // Setup ask for containers from RM
+ // Send request for containers to RM
+ // Until we get our fully allocated quota, we keep on polling RM for
+ // containers
+ // Keep looping until all the containers are launched and shell script
+ // executed on them ( regardless of success/failure).
+ ContainerRequest containerAsk = setupContainerAskForRM(numTotalContainers);
+ resourceManager.addContainerRequest(containerAsk);
+ numRequestedContainers.set(numTotalContainers);
- // Check what the current available resources in the cluster are
- // TODO should we do anything if the available resources are not enough?
- Resource availableResources = amResp.getAvailableResources();
- LOG.info("Current available resources in the cluster "
- + availableResources);
-
- // Check the completed containers
- List<ContainerStatus> completedContainers = amResp
- .getCompletedContainersStatuses();
- LOG.info("Got response from RM for container ask, completedCnt="
- + completedContainers.size());
- for (ContainerStatus containerStatus : completedContainers) {
- LOG.info("Got container status for containerID="
- + containerStatus.getContainerId() + ", state="
- + containerStatus.getState() + ", exitStatus="
- + containerStatus.getExitStatus() + ", diagnostics="
- + containerStatus.getDiagnostics());
-
- // non complete containers should not be here
- assert (containerStatus.getState() == ContainerState.COMPLETE);
-
- // increment counters for completed/failed containers
- int exitStatus = containerStatus.getExitStatus();
- if (0 != exitStatus) {
- // container failed
- if (-100 != exitStatus) {
- // shell script failed
- // counts as completed
- numCompletedContainers.incrementAndGet();
- numFailedContainers.incrementAndGet();
- } else {
- // something else bad happened
- // app job did not complete for some reason
- // we should re-try as the container was lost for some reason
- numAllocatedContainers.decrementAndGet();
- numRequestedContainers.decrementAndGet();
- // we do not need to release the container as it would be done
- // by the RM/CM.
- }
- } else {
- // nothing to do
- // container completed successfully
+ while (!done) {
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException ex) {}
+ }
+ finish();
+
+ return success;
+ }
+
+ private void finish() {
+ // Join all launched threads
+ // needed for when we time out
+ // and we need to release containers
+ for (Thread launchThread : launchThreads) {
+ try {
+ launchThread.join(10000);
+ } catch (InterruptedException e) {
+ LOG.info("Exception thrown in thread join: " + e.getMessage());
+ e.printStackTrace();
+ }
+ }
+
+ // When the application completes, it should send a finish application
+ // signal to the RM
+ LOG.info("Application completed. Signalling finish to RM");
+
+ FinalApplicationStatus appStatus;
+ String appMessage = null;
+ success = true;
+ if (numFailedContainers.get() == 0) {
+ appStatus = FinalApplicationStatus.SUCCEEDED;
+ } else {
+ appStatus = FinalApplicationStatus.FAILED;
+ appMessage = "Diagnostics." + ", total=" + numTotalContainers
+ + ", completed=" + numCompletedContainers.get() + ", allocated="
+ + numAllocatedContainers.get() + ", failed="
+ + numFailedContainers.get();
+ success = false;
+ }
+ try {
+ resourceManager.unregisterApplicationMaster(appStatus, appMessage, null);
+ } catch (YarnRemoteException ex) {
+ LOG.error("Failed to unregister application", ex);
+ }
+
+ done = true;
+ resourceManager.stop();
+ }
+
+ private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler {
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> completedContainers) {
+ LOG.info("Got response from RM for container ask, completedCnt="
+ + completedContainers.size());
+ for (ContainerStatus containerStatus : completedContainers) {
+ LOG.info("Got container status for containerID="
+ + containerStatus.getContainerId() + ", state="
+ + containerStatus.getState() + ", exitStatus="
+ + containerStatus.getExitStatus() + ", diagnostics="
+ + containerStatus.getDiagnostics());
+
+ // non complete containers should not be here
+ assert (containerStatus.getState() == ContainerState.COMPLETE);
+
+ // increment counters for completed/failed containers
+ int exitStatus = containerStatus.getExitStatus();
+ if (0 != exitStatus) {
+ // container failed
+ if (YarnConfiguration.ABORTED_CONTAINER_EXIT_STATUS != exitStatus) {
+ // shell script failed
+ // counts as completed
numCompletedContainers.incrementAndGet();
- LOG.info("Container completed successfully." + ", containerId="
- + containerStatus.getContainerId());
+ numFailedContainers.incrementAndGet();
+ } else {
+ // container was killed by framework, possibly preempted
+ // we should re-try as the container was lost for some reason
+ numAllocatedContainers.decrementAndGet();
+ numRequestedContainers.decrementAndGet();
+ // we do not need to release the container as it would be done
+ // by the RM
}
+ } else {
+ // nothing to do
+ // container completed successfully
+ numCompletedContainers.incrementAndGet();
+ LOG.info("Container completed successfully." + ", containerId="
+ + containerStatus.getContainerId());
}
- if (numCompletedContainers.get() == numTotalContainers) {
- appDone = true;
- }
-
- LOG.info("Current application state: loop=" + loopCounter
- + ", appDone=" + appDone + ", total=" + numTotalContainers
- + ", requested=" + numRequestedContainers + ", completed="
- + numCompletedContainers + ", failed=" + numFailedContainers
- + ", currentAllocated=" + numAllocatedContainers);
-
- // TODO
- // Add a timeout handling layer
- // for misbehaving shell commands
}
-
- // Join all launched threads
- // needed for when we time out
- // and we need to release containers
- for (Thread launchThread : launchThreads) {
- try {
- launchThread.join(10000);
- } catch (InterruptedException e) {
- LOG.info("Exception thrown in thread join: " + e.getMessage());
- e.printStackTrace();
- }
+
+ // ask for more containers if any failed
+ int askCount = numTotalContainers - numRequestedContainers.get();
+ numRequestedContainers.addAndGet(askCount);
+
+ if (askCount > 0) {
+ ContainerRequest containerAsk = setupContainerAskForRM(askCount);
+ resourceManager.addContainerRequest(containerAsk);
+ }
+
+ // set progress to deliver to RM on next heartbeat
+ float progress = (float) numCompletedContainers.get()
+ / numTotalContainers;
+ resourceManager.setProgress(progress);
+
+ if (numCompletedContainers.get() == numTotalContainers) {
+ done = true;
}
+ }
- // When the application completes, it should send a finish application
- // signal to the RM
- LOG.info("Application completed. Signalling finish to RM");
-
- FinalApplicationStatus appStatus;
- String appMessage = null;
- boolean isSuccess = true;
- if (numFailedContainers.get() == 0) {
- appStatus = FinalApplicationStatus.SUCCEEDED;
- } else {
- appStatus = FinalApplicationStatus.FAILED;
- appMessage = "Diagnostics." + ", total=" + numTotalContainers
- + ", completed=" + numCompletedContainers.get() + ", allocated="
- + numAllocatedContainers.get() + ", failed="
- + numFailedContainers.get();
- isSuccess = false;
+ @Override
+ public void onContainersAllocated(List<Container> allocatedContainers) {
+ LOG.info("Got response from RM for container ask, allocatedCnt="
+ + allocatedContainers.size());
+ numAllocatedContainers.addAndGet(allocatedContainers.size());
+ for (Container allocatedContainer : allocatedContainers) {
+ LOG.info("Launching shell command on a new container."
+ + ", containerId=" + allocatedContainer.getId()
+ + ", containerNode=" + allocatedContainer.getNodeId().getHost()
+ + ":" + allocatedContainer.getNodeId().getPort()
+ + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress()
+ + ", containerState" + allocatedContainer.getState()
+ + ", containerResourceMemory"
+ + allocatedContainer.getResource().getMemory());
+ // + ", containerToken"
+ // +allocatedContainer.getContainerToken().getIdentifier().toString());
+
+ LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(
+ allocatedContainer);
+ Thread launchThread = new Thread(runnableLaunchContainer);
+
+ // launch and start the container on a separate thread to keep
+ // the main thread unblocked
+ // as all containers may not be allocated at one go.
+ launchThreads.add(launchThread);
+ launchThread.start();
}
- resourceManager.unregisterApplicationMaster(appStatus, appMessage, null);
- return isSuccess;
- } finally {
- resourceManager.stop();
}
+
+ @Override
+ public void onRebootRequest() {}
+
+ @Override
+ public void onNodesUpdated(List<NodeReport> updatedNodes) {}
}
/**
@@ -811,22 +792,4 @@ public class ApplicationMaster {
LOG.info("Requested container ask: " + request.toString());
return request;
}
-
- /**
- * Ask RM to allocate given no. of containers to this Application Master
- *
- * @param requestedContainers Containers to ask for from RM
- * @return Response from RM to AM with allocated containers
- * @throws YarnRemoteException
- */
- private AMResponse sendContainerAskToRM() throws YarnRemoteException {
- float progressIndicator = (float) numCompletedContainers.get()
- / numTotalContainers;
-
- LOG.info("Sending request to RM for containers" + ", progress="
- + progressIndicator);
-
- AllocateResponse resp = resourceManager.allocate(progressIndicator);
- return resp.getAMResponse();
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Sun Mar 24 15:56:18 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.applications.distributedshell;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
@@ -481,14 +483,15 @@ public class Client extends YarnClientIm
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar
- StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
+ StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$())
+ .append(File.pathSeparatorChar).append("./*");
for (String c : conf.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- classPathEnv.append(':');
+ classPathEnv.append(File.pathSeparatorChar);
classPathEnv.append(c.trim());
}
- classPathEnv.append(":./log4j.properties");
+ classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
// add the runtime classpath needed for tests to work
if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
@@ -505,7 +508,7 @@ public class Client extends YarnClientIm
// Set java executable command
LOG.info("Setting up app master command");
- vargs.add("${JAVA_HOME}" + "/bin/java");
+ vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
// Set Xmx based on am memory size
vargs.add("-Xmx" + amMemory + "m");
// Set class name
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Sun Mar 24 15:56:18 2013
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -54,8 +55,8 @@ public class TestDistributedShell {
conf.setClass(YarnConfiguration.RM_SCHEDULER,
FifoScheduler.class, ResourceScheduler.class);
if (yarnCluster == null) {
- yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getName(),
- 1, 1, 1);
+ yarnCluster = new MiniYARNCluster(
+ TestDistributedShell.class.getSimpleName(), 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
@@ -82,7 +83,7 @@ public class TestDistributedShell {
}
}
- @Test
+ @Test(timeout=30000)
public void testDSShell() throws Exception {
String[] args = {
@@ -91,7 +92,7 @@ public class TestDistributedShell {
"--num_containers",
"2",
"--shell_command",
- "ls",
+ Shell.WINDOWS ? "dir" : "ls",
"--master_memory",
"512",
"--container_memory",
@@ -110,7 +111,7 @@ public class TestDistributedShell {
}
- @Test
+ @Test(timeout=30000)
public void testDSShellWithNoArgs() throws Exception {
String[] args = {};
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java Sun Mar 24 15:56:18 2013
@@ -91,7 +91,7 @@ public class TestUnmanagedAMLauncher {
return envClassPath;
}
- @Test
+ @Test(timeout=10000)
public void testDSShell() throws Exception {
String classpath = getTestRuntimeClasspath();
String javaHome = System.getenv("JAVA_HOME");
@@ -110,7 +110,7 @@ public class TestUnmanagedAMLauncher {
javaHome
+ "/bin/java -Xmx512m "
+ "org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster "
- + "--container_memory 128 --num_containers 0 --priority 0 --shell_command ls" };
+ + "--container_memory 128 --num_containers 1 --priority 0 --shell_command ls" };
LOG.info("Initializing Launcher");
UnmanagedAMLauncher launcher = new UnmanagedAMLauncher(new Configuration(
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java Sun Mar 24 15:56:18 2013
@@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -194,13 +193,12 @@ public class AMRMClientImpl extends Abst
}
allocateResponse = rmClient.allocate(allocateRequest);
- AMResponse response = allocateResponse.getAMResponse();
synchronized (this) {
// update these on successful RPC
clusterNodeCount = allocateResponse.getNumClusterNodes();
- lastResponseId = response.getResponseId();
- clusterAvailableResources = response.getAvailableResources();
+ lastResponseId = allocateResponse.getResponseId();
+ clusterAvailableResources = allocateResponse.getAvailableResources();
}
} finally {
// TODO how to differentiate remote yarn exception vs error in rpc
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Sun Mar 24 15:56:18 2013
@@ -153,6 +153,10 @@ public class ApplicationCLI extends Yarn
appReportStr.println(appReport.getFinalApplicationStatus());
appReportStr.print("\tTracking-URL : ");
appReportStr.println(appReport.getOriginalTrackingUrl());
+ appReportStr.print("\tRPC Port : ");
+ appReportStr.println(appReport.getRpcPort());
+ appReportStr.print("\tAM Host : ");
+ appReportStr.println(appReport.getHost());
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java Sun Mar 24 15:56:18 2013
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -202,9 +201,8 @@ public class TestAMRMClient {
assertTrue(amClient.release.size() == 0);
assertTrue(nodeCount == amClient.getClusterNodeCount());
- AMResponse amResponse = allocResponse.getAMResponse();
- allocatedContainerCount += amResponse.getAllocatedContainers().size();
- for(Container container : amResponse.getAllocatedContainers()) {
+ allocatedContainerCount += allocResponse.getAllocatedContainers().size();
+ for(Container container : allocResponse.getAllocatedContainers()) {
ContainerId rejectContainerId = container.getId();
releases.add(rejectContainerId);
amClient.releaseAssignedContainer(rejectContainerId);
@@ -264,11 +262,11 @@ public class TestAMRMClient {
while(!releases.isEmpty() || iterationsLeft-- > 0) {
// inform RM of rejection
AllocateResponse allocResponse = amClient.allocate(0.1f);
- AMResponse amResponse = allocResponse.getAMResponse();
// RM did not send new containers because AM does not need any
- assertTrue(amResponse.getAllocatedContainers().size() == 0);
- if(amResponse.getCompletedContainersStatuses().size() > 0) {
- for(ContainerStatus cStatus : amResponse.getCompletedContainersStatuses()) {
+ assertTrue(allocResponse.getAllocatedContainers().size() == 0);
+ if(allocResponse.getCompletedContainersStatuses().size() > 0) {
+ for(ContainerStatus cStatus :allocResponse
+ .getCompletedContainersStatuses()) {
if(releases.contains(cStatus.getContainerId())) {
assertTrue(cStatus.getState() == ContainerState.COMPLETE);
assertTrue(cStatus.getExitStatus() == -100);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Sun Mar 24 15:56:18 2013
@@ -94,6 +94,8 @@ public class TestYarnCLI {
pw.println("\tState : FINISHED");
pw.println("\tFinal-State : SUCCEEDED");
pw.println("\tTracking-URL : N/A");
+ pw.println("\tRPC Port : 124");
+ pw.println("\tAM Host : host");
pw.println("\tDiagnostics : diagnostics");
pw.close();
String appReportStr = baos.toString("UTF-8");
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sun Mar 24 15:56:18 2013
@@ -437,6 +437,16 @@ public class YarnConfiguration extends C
public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
public static final int DEFAULT_NM_PMEM_MB = 8 * 1024;
+ /** Specifies whether physical memory check is enabled. */
+ public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX
+ + "pmem-check-enabled";
+ public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true;
+
+ /** Specifies whether physical memory check is enabled. */
+ public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX
+ + "vmem-check-enabled";
+ public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true;
+
/** Conversion ratio for physical memory to virtual memory. */
public static final String NM_VMEM_PMEM_RATIO =
NM_PREFIX + "vmem-pmem-ratio";
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java Sun Mar 24 15:56:18 2013
@@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.Writer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -505,7 +506,7 @@ public class AggregatedLogFormat {
* @throws IOException
*/
public static void readAContainerLogsForALogType(
- DataInputStream valueStream, DataOutputStream out)
+ DataInputStream valueStream, PrintStream out)
throws IOException {
byte[] buf = new byte[65535];
@@ -513,11 +514,11 @@ public class AggregatedLogFormat {
String fileType = valueStream.readUTF();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
- out.writeUTF("\nLogType:");
- out.writeUTF(fileType);
- out.writeUTF("\nLogLength:");
- out.writeUTF(fileLengthStr);
- out.writeUTF("\nLog Contents:\n");
+ out.print("LogType: ");
+ out.println(fileType);
+ out.print("LogLength: ");
+ out.println(fileLengthStr);
+ out.println("Log Contents:");
int curRead = 0;
long pendingRead = fileLength - curRead;
@@ -533,6 +534,7 @@ public class AggregatedLogFormat {
pendingRead > buf.length ? buf.length : (int) pendingRead;
len = valueStream.read(buf, 0, toRead);
}
+ out.println("");
}
public void close() throws IOException {