You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/09/14 00:49:38 UTC
svn commit: r1170378 [4/12] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-clie...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java Tue Sep 13 22:49:27 2011
@@ -18,13 +18,131 @@
package org.apache.hadoop.yarn.api.records;
-public interface ContainerId extends Comparable<ContainerId>{
- public abstract ApplicationAttemptId getAppAttemptId();
- public abstract ApplicationId getAppId();
- public abstract int getId();
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p><code>ContainerId</code> represents a globally unique identifier
+ * for a {@link Container} in the cluster.</p>
+ */
+@Public
+@Stable
+public abstract class ContainerId implements Comparable<ContainerId>{
+ /**
+ * Get the <code>ApplicationAttemptId</code> of the application to which
+ * the <code>Container</code> was assigned.
+ * @return <code>ApplicationAttemptId</code> of the application to which
+ * the <code>Container</code> was assigned
+ */
+ @Public
+ @Stable
+ public abstract ApplicationAttemptId getApplicationAttemptId();
- public abstract void setAppAttemptId(ApplicationAttemptId atId);
- public abstract void setAppId(ApplicationId appID);
+ @Private
+ @Unstable
+ public abstract void setApplicationAttemptId(ApplicationAttemptId atId);
+
+ /**
+ * Get the identifier of the <code>ContainerId</code>.
+ * @return identifier of the <code>ContainerId</code>
+ */
+ @Public
+ @Stable
+ public abstract int getId();
+
+ @Private
+ @Unstable
public abstract void setId(int id);
+
+
+ // TODO: Why thread local?
+ // ^ NumberFormat instances are not threadsafe
+ private static final ThreadLocal<NumberFormat> appIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(4);
+ return fmt;
+ }
+ };
+
+ // TODO: fail the app submission if attempts are more than 10 or something
+ private static final ThreadLocal<NumberFormat> appAttemptIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(2);
+ return fmt;
+ }
+ };
+ // TODO: Why thread local?
+ // ^ NumberFormat instances are not threadsafe
+ private static final ThreadLocal<NumberFormat> containerIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(6);
+ return fmt;
+ }
+ };
+
+ @Override
+ public int hashCode() {
+ // Generated by eclipse.
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + getId();
+ result = prime * result
+ + ((getApplicationAttemptId() == null) ? 0 : getApplicationAttemptId().hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ ContainerId otherCId = (ContainerId)other;
+ if (this.getApplicationAttemptId().equals(
+ otherCId.getApplicationAttemptId())) {
+ return this.getId() == otherCId.getId();
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(ContainerId other) {
+ if (this.getApplicationAttemptId().compareTo(
+ other.getApplicationAttemptId()) == 0) {
+ return this.getId() - other.getId();
+ } else {
+ return this.getApplicationAttemptId().compareTo(
+ other.getApplicationAttemptId());
+ }
+
+ }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ ApplicationId appId = getApplicationAttemptId().getApplicationId();
+ sb.append("container_").append(appId.getClusterTimestamp()).append("_");
+ sb.append(appIdFormat.get().format(appId.getId())).append("_");
+ sb.append(appAttemptIdFormat.get().format(getApplicationAttemptId().
+ getAttemptId())).append("_");
+ sb.append(containerIdFormat.get().format(getId()));
+ return sb.toString();
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java Tue Sep 13 22:49:27 2011
@@ -22,10 +22,8 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerManager;
/**
@@ -121,100 +119,52 @@ public interface ContainerLaunchContext
void setContainerTokens(ByteBuffer containerToken);
/**
- * Get all <code>LocalResource</code> required by the container.
+ * Get <code>LocalResource</code> required by the container.
* @return all <code>LocalResource</code> required by the container
*/
@Public
@Stable
- Map<String, LocalResource> getAllLocalResources();
+ Map<String, LocalResource> getLocalResources();
- @Private
- @Unstable
- LocalResource getLocalResource(String key);
-
/**
- * Add all <code>LocalResource</code> required by the container.
+ * Set <code>LocalResource</code> required by the container.
* @param localResources <code>LocalResource</code> required by the container
*/
@Public
@Stable
- void addAllLocalResources(Map<String, LocalResource> localResources);
-
- @Private
- @Unstable
- void setLocalResource(String key, LocalResource value);
-
- @Private
- @Unstable
- void removeLocalResource(String key);
-
- @Private
- @Unstable
- void clearLocalResources();
+ void setLocalResources(Map<String, LocalResource> localResources);
/**
- * Get application-specific binary service data.
- * @return application-specific binary service data
+ * Get application-specific binary <em>service data</em>.
+ * @return application-specific binary <em>service data</em>
*/
@Public
@Stable
- Map<String, ByteBuffer> getAllServiceData();
+ Map<String, ByteBuffer> getServiceData();
- @Private
- @Unstable
- ByteBuffer getServiceData(String key);
-
/**
- * Add add application-specific binary service data.
- * @param serviceData application-specific binary service data
+ * Set application-specific binary <em>service data</em>.
+ * @param serviceData application-specific binary <em>service data</em>
*/
@Public
@Stable
- void addAllServiceData(Map<String, ByteBuffer> serviceData);
-
- @Private
- @Unstable
- void setServiceData(String key, ByteBuffer value);
-
- @Private
- @Unstable
- void removeServiceData(String key);
-
- @Private
- @Unstable
- void clearServiceData();
+ void setServiceData(Map<String, ByteBuffer> serviceData);
/**
- * Get <em>environment variables</em> for the launched container.
- * @return <em>environment variables</em> for the launched container
+ * Get <em>environment variables</em> for the container.
+ * @return <em>environment variables</em> for the container
*/
@Public
@Stable
- Map<String, String> getAllEnv();
-
- @Private
- @Unstable
- String getEnv(String key);
-
+ Map<String, String> getEnv();
+
/**
- * Add <em>environment variables</em> for the launched container.
- * @param env <em>environment variables</em> for the launched container
+ * Add <em>environment variables</em> for the container.
+ * @param environment <em>environment variables</em> for the container
*/
@Public
@Stable
- void addAllEnv(Map<String, String> env);
-
- @Private
- @Unstable
- void setEnv(String key, String value);
-
- @Private
- @Unstable
- void removeEnv(String key);
-
- @Private
- @Unstable
- void clearEnv();
+ void setEnv(Map<String, String> environment);
/**
* Get the list of <em>commands</em> for launching the container.
@@ -222,15 +172,7 @@ public interface ContainerLaunchContext
*/
@Public
@Stable
- List<String> getCommandList();
-
- @Private
- @Unstable
- String getCommand(int index);
-
- @Private
- @Unstable
- int getCommandCount();
+ List<String> getCommands();
/**
* Add the list of <em>commands</em> for launching the container.
@@ -238,17 +180,6 @@ public interface ContainerLaunchContext
*/
@Public
@Stable
- void addAllCommands(List<String> commands);
-
- @Private
- @Unstable
- void addCommand(String command);
-
- @Private
- @Unstable
- void removeCommand(int index);
+ void setCommands(List<String> commands);
- @Private
- @Unstable
- void clearCommands();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java Tue Sep 13 22:49:27 2011
@@ -18,6 +18,16 @@
package org.apache.hadoop.yarn.api.records;
+/**
+ * <p>State of a <code>Container</code>.</p>
+ */
public enum ContainerState {
- NEW, RUNNING, COMPLETE
+ /** New container */
+ NEW,
+
+ /** Running container */
+ RUNNING,
+
+ /** Completed container */
+ COMPLETE
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java Tue Sep 13 22:49:27 2011
@@ -18,14 +18,81 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p><code>ContainerStatus</code> represents the current status of a
+ * <code>Container</code>.</p>
+ *
+ * <p>It provides details such as:
+ * <ul>
+ * <li><code>ContainerId</code> of the container.</li>
+ * <li><code>ContainerState</code> of the container.</li>
+ * <li><em>Exit status</em> of a completed container.</li>
+ * <li><em>Diagnostic</em> message for a failed container.</li>
+ * </ul>
+ * </p>
+ */
+@Public
+@Stable
public interface ContainerStatus {
+ /**
+ * Get the <code>ContainerId</code> of the container.
+ * @return <code>ContainerId</code> of the container
+ */
+ @Public
+ @Stable
ContainerId getContainerId();
- ContainerState getState();
- String getExitStatus();
- String getDiagnostics();
+ @Private
+ @Unstable
void setContainerId(ContainerId containerId);
+
+ /**
+ * Get the <code>ContainerState</code> of the container.
+ * @return <code>ContainerState</code> of the container
+ */
+ @Public
+ @Stable
+ ContainerState getState();
+
+ @Private
+ @Unstable
void setState(ContainerState state);
- void setExitStatus(String exitStatus);
+
+ /**
+ * <p>Get the <em>exit status</em> for the container.</p>
+ *
+ * <p>Note: This is valid only for completed containers i.e. containers
+ * with state {@link ContainerState#COMPLETE}.
+ * Otherwise, it returns an invalid exit code equal to {@literal -1000};</p>
+ *
+ * <p>Container killed by the framework, either due to being released by
+ * the application or being 'lost' due to node failures etc. have a special
+ * exit code of {@literal -100}.</p>
+ *
+ * @return <em>exit status</em> for the container
+ */
+ @Public
+ @Stable
+ int getExitStatus();
+
+ @Private
+ @Unstable
+ void setExitStatus(int exitStatus);
+
+ /**
+ * Get <em>diagnostic messages</em> for failed containers.
+ * @return <em>diagnostic messages</em> for failed containers
+ */
+ @Public
+ @Stable
+ String getDiagnostics();
+
+ @Private
+ @Unstable
void setDiagnostics(String diagnostics);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java Tue Sep 13 22:49:27 2011
@@ -20,15 +20,76 @@ package org.apache.hadoop.yarn.api.recor
import java.nio.ByteBuffer;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ContainerManager;
+
+/**
+ * <p><code>ContainerToken</code> is the security token used by the framework
+ * to verify authenticity of any <code>Container</code>.</p>
+ *
+ * <p>The <code>ResourceManager</code>, on container allocation provides a
+ * secure token which is verified by the <code>NodeManager</code> on
+ * container launch.</p>
+ *
+ * <p>Applications do not need to care about <code>ContainerToken</code>, they
+ * are transparently handled by the framework - the allocated
+ * <code>Container</code> includes the <code>ContainerToken</code>.</p>
+ *
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ */
+@Public
+@Stable
public interface ContainerToken {
+ /**
+ * Get the token identifier.
+ * @return token identifier
+ */
+ @Public
+ @Stable
public abstract ByteBuffer getIdentifier();
- public abstract ByteBuffer getPassword();
- public abstract String getKind();
- public abstract String getService();
+ @Private
+ @Stable
public abstract void setIdentifier(ByteBuffer identifier);
+
+ /**
+ * Get the token password
+ * @return token password
+ */
+ @Public
+ @Stable
+ public abstract ByteBuffer getPassword();
+
+ @Private
+ @Stable
public abstract void setPassword(ByteBuffer password);
+
+ /**
+ * Get the token kind.
+ * @return token kind
+ */
+ @Public
+ @Stable
+ public abstract String getKind();
+
+ @Private
+ @Stable
public abstract void setKind(String kind);
+
+ /**
+ * Get the service to which the token is allocated.
+ * @return service to which the token is allocated
+ */
+ @Public
+ @Stable
+ public abstract String getService();
+
+ @Private
+ @Stable
public abstract void setService(String service);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java Tue Sep 13 22:49:27 2011
@@ -18,11 +18,43 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p><code>NodeId</code> is the unique identifier for a node.</p>
+ *
+ * <p>It includes the <em>hostname</em> and <em>port</em> to uniquely
+ * identify the node. Thus, it is unique across restarts of any
+ * <code>NodeManager</code>.</p>
+ */
+@Public
+@Stable
public interface NodeId extends Comparable<NodeId> {
+ /**
+ * Get the <em>hostname</em> of the node.
+ * @return <em>hostname</em> of the node
+ */
+ @Public
+ @Stable
String getHost();
+
+ @Private
+ @Unstable
void setHost(String host);
+ /**
+ * Get the <em>port</em> for communicating with the node.
+ * @return <em>port</em> for communicating with the node
+ */
+ @Public
+ @Stable
int getPort();
+
+ @Private
+ @Unstable
void setPort(int port);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProtoBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProtoBase.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProtoBase.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProtoBase.java Tue Sep 13 22:49:27 2011
@@ -20,11 +20,15 @@ package org.apache.hadoop.yarn.api.recor
import java.nio.ByteBuffer;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.util.ProtoUtils;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
+@Private
+@Unstable
public abstract class ProtoBase <T extends Message> {
public abstract T getProto();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java Tue Sep 13 22:49:27 2011
@@ -18,10 +18,40 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+
+/**
+ * <p><code>Resource</code> models a set of computer resources in the
+ * cluster.</p>
+ *
+ * <p>Currrently it only models <em>memory</em>.</p>
+ *
+ * <p>Typically, applications request <code>Resource</code> of suitable
+ * capability to run their component tasks.</p>
+ *
+ * @see ResourceRequest
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ */
+@Public
+@Stable
public interface Resource extends Comparable<Resource> {
+
+ /**
+ * Get <em>memory</em> of the resource.
+ * @return <em>memory</em> of the resource
+ */
+ @Public
+ @Stable
public abstract int getMemory();
+ /**
+ * Set <em>memory</em> of the resource.
+ * @param memory <em>memory</em> of the resource
+ */
+ @Public
+ @Stable
public abstract void setMemory(int memory);
-
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java Tue Sep 13 22:49:27 2011
@@ -18,16 +18,107 @@
package org.apache.hadoop.yarn.api.records;
-public interface ResourceRequest extends Comparable<ResourceRequest>{
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+
+/**
+ * <p><code>ResourceRequest</code> represents the request made by an
+ * application to the <code>ResourceManager</code> to obtain various
+ * <code>Container</code> allocations.</p>
+ *
+ * <p>It includes:
+ * <ul>
+ * <li>{@link Priority} of the request.</li>
+ * <li>
+ * The <em>name</em> of the machine or rack on which the allocation is
+ * desired. A special value of <em>*</em> signifies that
+ * <em>any</em> host/rack is acceptable to the application.
+ * </li>
+ * <li>{@link Resource} required for each request.</li>
+ * <li>
+ * Number of containers of such specifications which are required
+ * by the application.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * @see Resource
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ */
+@Public
+@Stable
+public interface ResourceRequest extends Comparable<ResourceRequest> {
+ /**
+ * Get the <code>Priority</code> of the request.
+ * @return <code>Priority</code> of the request
+ */
+ @Public
+ @Stable
public abstract Priority getPriority();
+
+ /**
+ * Set the <code>Priority</code> of the request
+ * @param priority <code>Priority</code> of the request
+ */
+ @Public
+ @Stable
+ public abstract void setPriority(Priority priority);
+
+ /**
+ * Get the <em>host/rack</em> on which the allocation is desired.
+ *
+ * A special value of <em>*</em> signifies that <em>any</em> host/rack is
+ * acceptable.
+ *
+ * @return <em>host/rack</em> on which the allocation is desired
+ */
+ @Public
+ @Stable
public abstract String getHostName();
+
+ /**
+ * Set <em>host/rack</em> on which the allocation is desired.
+ *
+ * A special value of <em>*</em> signifies that <em>any</em> host/rack is
+ * acceptable.
+ *
+ * @param hostName <em>host/rack</em> on which the allocation is desired
+ */
+ @Public
+ @Stable
+ public abstract void setHostName(String hostName);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
public abstract Resource getCapability();
- public abstract int getNumContainers();
- public abstract void setPriority(Priority priority);
- public abstract void setHostName(String hostName);
+ /**
+ * Set the <code>Resource</code> capability of the request
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
public abstract void setCapability(Resource capability);
- public abstract void setNumContainers(int numContainers);
-
+ /**
+ * Get the number of containers required with the given specifications.
+ * @return number of containers required with the given specifications
+ */
+ @Public
+ @Stable
+ public abstract int getNumContainers();
+
+ /**
+ * Set the number of containers required with the given specifications
+ * @param numContainers number of containers required with the given
+ * specifications
+ */
+ @Public
+ @Stable
+ public abstract void setNumContainers(int numContainers);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java Tue Sep 13 22:49:27 2011
@@ -18,14 +18,77 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * <p><code>URL</code> represents a serializable {@link java.net.URL}.</p>
+ */
+@Public
+@Evolving
public interface URL {
+
+ /**
+ * Get the scheme of the URL.
+ * @return scheme of the URL
+ */
+ @Public
+ @Evolving
public abstract String getScheme();
- public abstract String getHost();
- public abstract int getPort();
- public abstract String getFile();
+ /**
+ * Set the scheme of the URL
+ * @param scheme scheme of the URL
+ */
+ @Public
+ @Evolving
public abstract void setScheme(String scheme);
+
+ /**
+ * Get the host of the URL.
+ * @return host of the URL
+ */
+ @Public
+ @Evolving
+ public abstract String getHost();
+
+ /**
+ * Set the host of the URL.
+ * @param host host of the URL
+ */
+ @Public
+ @Evolving
public abstract void setHost(String host);
+
+ /**
+ * Get the port of the URL.
+ * @return port of the URL
+ */
+ @Public
+ @Evolving
+ public abstract int getPort();
+
+ /**
+ * Set the port of the URL
+ * @param port port of the URL
+ */
+ @Public
+ @Evolving
public abstract void setPort(int port);
+
+ /**
+ * Get the file of the URL.
+ * @return file of the URL
+ */
+ @Public
+ @Evolving
+ public abstract String getFile();
+
+ /**
+ * Set the file of the URL.
+ * @param file file of the URL
+ */
+ @Public
+ @Evolving
public abstract void setFile(String file);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java Tue Sep 13 22:49:27 2011
@@ -18,10 +18,30 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p><code>YarnClusterMetrics</code> represents cluster metrics.</p>
+ *
+ * <p>Currently only number of <code>NodeManager</code>s is provided.</p>
+ */
+@Public
+@Stable
public interface YarnClusterMetrics {
+ /**
+ * Get the number of <code>NodeManager</code>s in the cluster.
+ * @return number of <code>NodeManager</code>s in the cluster
+ */
+ @Public
+ @Stable
public abstract int getNumNodeManagers();
-
+
+ @Private
+ @Unstable
public abstract void setNumNodeManagers(int numNodeManagers);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java Tue Sep 13 22:49:27 2011
@@ -25,11 +25,13 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProto;
import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -41,8 +43,8 @@ public class AMResponsePBImpl extends Pr
Resource limit;
- private List<Container> newContainersList = null;
- private List<Container> finishedContainersList = null;
+ private List<Container> allocatedContainers = null;
+ private List<ContainerStatus> completedContainersStatuses = null;
// private boolean hasLocalContainerList = false;
@@ -63,15 +65,17 @@ public class AMResponsePBImpl extends Pr
}
private synchronized void mergeLocalToBuilder() {
- if (this.newContainersList != null) {
- builder.clearNewContainers();
- Iterable<ContainerProto> iterable = getProtoIterable(this.newContainersList);
- builder.addAllNewContainers(iterable);
+ if (this.allocatedContainers != null) {
+ builder.clearAllocatedContainers();
+ Iterable<ContainerProto> iterable =
+ getProtoIterable(this.allocatedContainers);
+ builder.addAllAllocatedContainers(iterable);
}
- if (this.finishedContainersList != null) {
- builder.clearFinishedContainers();
- Iterable<ContainerProto> iterable = getProtoIterable(this.finishedContainersList);
- builder.addAllFinishedContainers(iterable);
+ if (this.completedContainersStatuses != null) {
+ builder.clearCompletedContainerStatuses();
+ Iterable<ContainerStatusProto> iterable =
+ getContainerStatusProtoIterable(this.completedContainersStatuses);
+ builder.addAllCompletedContainerStatuses(iterable);
}
if (this.limit != null) {
builder.setLimit(convertToProtoFormat(this.limit));
@@ -139,42 +143,31 @@ public class AMResponsePBImpl extends Pr
}
@Override
- public synchronized List<Container> getNewContainerList() {
+ public synchronized List<Container> getAllocatedContainers() {
initLocalNewContainerList();
- return this.newContainersList;
- }
-
- @Override
- public synchronized Container getNewContainer(int index) {
- initLocalNewContainerList();
- return this.newContainersList.get(index);
- }
- @Override
- public synchronized int getNewContainerCount() {
- initLocalNewContainerList();
- return this.newContainersList.size();
+ return this.allocatedContainers;
}
//Once this is called. containerList will never be null - untill a getProto is called.
private synchronized void initLocalNewContainerList() {
- if (this.newContainersList != null) {
+ if (this.allocatedContainers != null) {
return;
}
AMResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerProto> list = p.getNewContainersList();
- newContainersList = new ArrayList<Container>();
+ List<ContainerProto> list = p.getAllocatedContainersList();
+ allocatedContainers = new ArrayList<Container>();
for (ContainerProto c : list) {
- newContainersList.add(convertFromProtoFormat(c));
+ allocatedContainers.add(convertFromProtoFormat(c));
}
}
@Override
- public synchronized void addAllNewContainers(final List<Container> containers) {
+ public synchronized void setAllocatedContainers(final List<Container> containers) {
if (containers == null)
return;
initLocalNewContainerList();
- newContainersList.addAll(containers);
+ allocatedContainers.addAll(containers);
}
private synchronized Iterable<ContainerProto> getProtoIterable(
@@ -207,86 +200,71 @@ public class AMResponsePBImpl extends Pr
}
};
}
-
- @Override
- public synchronized void addNewContainer(Container containers) {
- initLocalNewContainerList();
- if (containers == null)
- return;
- this.newContainersList.add(containers);
- }
-
- @Override
- public synchronized void removeNewContainer(int index) {
- initLocalNewContainerList();
- this.newContainersList.remove(index);
- }
- @Override
- public synchronized void clearNewContainers() {
- initLocalNewContainerList();
- this.newContainersList.clear();
+
+ private synchronized Iterable<ContainerStatusProto>
+ getContainerStatusProtoIterable(
+ final List<ContainerStatus> newContainersList) {
+ maybeInitBuilder();
+ return new Iterable<ContainerStatusProto>() {
+ @Override
+ public synchronized Iterator<ContainerStatusProto> iterator() {
+ return new Iterator<ContainerStatusProto>() {
+
+ Iterator<ContainerStatus> iter = newContainersList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized ContainerStatusProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
}
//// Finished containers
@Override
- public synchronized List<Container> getFinishedContainerList() {
+ public synchronized List<ContainerStatus> getCompletedContainersStatuses() {
initLocalFinishedContainerList();
- return this.finishedContainersList;
- }
-
- @Override
- public synchronized Container getFinishedContainer(int index) {
- initLocalFinishedContainerList();
- return this.finishedContainersList.get(index);
- }
- @Override
- public synchronized int getFinishedContainerCount() {
- initLocalFinishedContainerList();
- return this.finishedContainersList.size();
+ return this.completedContainersStatuses;
}
//Once this is called. containerList will never be null - untill a getProto is called.
private synchronized void initLocalFinishedContainerList() {
- if (this.finishedContainersList != null) {
+ if (this.completedContainersStatuses != null) {
return;
}
AMResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerProto> list = p.getFinishedContainersList();
- finishedContainersList = new ArrayList<Container>();
+ List<ContainerStatusProto> list = p.getCompletedContainerStatusesList();
+ completedContainersStatuses = new ArrayList<ContainerStatus>();
- for (ContainerProto c : list) {
- finishedContainersList.add(convertFromProtoFormat(c));
+ for (ContainerStatusProto c : list) {
+ completedContainersStatuses.add(convertFromProtoFormat(c));
}
}
@Override
- public synchronized void addAllFinishedContainers(final List<Container> containers) {
+ public synchronized void setCompletedContainersStatuses(
+ final List<ContainerStatus> containers) {
if (containers == null)
return;
initLocalFinishedContainerList();
- finishedContainersList.addAll(containers);
+ completedContainersStatuses.addAll(containers);
}
- @Override
- public synchronized void addFinishedContainer(Container containers) {
- initLocalFinishedContainerList();
- if (containers == null)
- return;
- this.finishedContainersList.add(containers);
- }
-
- @Override
- public synchronized void removeFinishedContainer(int index) {
- initLocalFinishedContainerList();
- this.finishedContainersList.remove(index);
- }
- @Override
- public synchronized void clearFinishedContainers() {
- initLocalFinishedContainerList();
- this.finishedContainersList.clear();
- }
-
- private synchronized ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
+ private synchronized ContainerPBImpl convertFromProtoFormat(
+ ContainerProto p) {
return new ContainerPBImpl(p);
}
@@ -294,6 +272,15 @@ public class AMResponsePBImpl extends Pr
return ((ContainerPBImpl)t).getProto();
}
+ private synchronized ContainerStatusPBImpl convertFromProtoFormat(
+ ContainerStatusProto p) {
+ return new ContainerStatusPBImpl(p);
+ }
+
+ private synchronized ContainerStatusProto convertToProtoFormat(ContainerStatus t) {
+ return ((ContainerStatusPBImpl)t).getProto();
+ }
+
private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java Tue Sep 13 22:49:27 2011
@@ -18,35 +18,19 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-
-import java.text.NumberFormat;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-public class ApplicationAttemptIdPBImpl extends ProtoBase<ApplicationAttemptIdProto> implements ApplicationAttemptId {
+public class ApplicationAttemptIdPBImpl extends ApplicationAttemptId {
ApplicationAttemptIdProto proto = ApplicationAttemptIdProto.getDefaultInstance();
ApplicationAttemptIdProto.Builder builder = null;
boolean viaProto = false;
private ApplicationId applicationId = null;
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(4);
- }
-
- protected static final NumberFormat counterFormat = NumberFormat.getInstance();
- static {
- counterFormat.setGroupingUsed(false);
- counterFormat.setMinimumIntegerDigits(6);
- }
-
-
+
public ApplicationAttemptIdPBImpl() {
builder = ApplicationAttemptIdProto.newBuilder();
}
@@ -117,44 +101,11 @@ public class ApplicationAttemptIdPBImpl
this.applicationId = appId;
}
- private synchronized ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
+ private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p);
}
- private synchronized ApplicationIdProto convertToProtoFormat(ApplicationId t) {
+ private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
return ((ApplicationIdPBImpl)t).getProto();
}
-
- @Override
- public synchronized int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public synchronized boolean equals(Object other) {
- if (other == null) return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public synchronized int compareTo(ApplicationAttemptId other) {
- int compareAppIds = this.getApplicationId().compareTo(
- other.getApplicationId());
- if (compareAppIds == 0) {
- return this.getAttemptId() - other.getAttemptId();
- } else {
- return compareAppIds;
- }
-
- }
-
- @Override
- public synchronized String toString() {
- String id = (this.getApplicationId() != null) ? this.getApplicationId().getClusterTimestamp() + "_" +
- idFormat.format(this.getApplicationId().getId()): "none";
- return "appattempt_" + id + "_" + counterFormat.format(getAttemptId());
- }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java Tue Sep 13 22:49:27 2011
@@ -20,13 +20,12 @@ package org.apache.hadoop.yarn.api.recor
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProtoOrBuilder;
-public class ApplicationIdPBImpl extends ProtoBase<ApplicationIdProto> implements ApplicationId {
+public class ApplicationIdPBImpl extends ApplicationId {
ApplicationIdProto proto = ApplicationIdProto.getDefaultInstance();
ApplicationIdProto.Builder builder = null;
boolean viaProto = false;
@@ -40,13 +39,13 @@ public class ApplicationIdPBImpl extends
viaProto = true;
}
- public ApplicationIdProto getProto() {
+ public synchronized ApplicationIdProto getProto() {
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationIdProto.newBuilder(proto);
}
@@ -55,40 +54,25 @@ public class ApplicationIdPBImpl extends
@Override
- public int getId() {
+ public synchronized int getId() {
ApplicationIdProtoOrBuilder p = viaProto ? proto : builder;
return (p.getId());
}
@Override
- public void setId(int id) {
+ public synchronized void setId(int id) {
maybeInitBuilder();
builder.setId((id));
}
@Override
- public long getClusterTimestamp() {
+ public synchronized long getClusterTimestamp() {
ApplicationIdProtoOrBuilder p = viaProto ? proto : builder;
return (p.getClusterTimestamp());
}
@Override
- public void setClusterTimestamp(long clusterTimestamp) {
+ public synchronized void setClusterTimestamp(long clusterTimestamp) {
maybeInitBuilder();
builder.setClusterTimestamp((clusterTimestamp));
}
-
- @Override
- public int compareTo(ApplicationId other) {
- if (this.getId() - other.getId() == 0) {
- return this.getClusterTimestamp() > other.getClusterTimestamp() ? 1 :
- this.getClusterTimestamp() < other.getClusterTimestamp() ? -1 : 0;
- } else {
- return this.getId() - other.getId();
- }
- }
-
- @Override
- public String toString() {
- return "application_" + this.getClusterTimestamp() + "_" + this.getId();
- }
-}
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java Tue Sep 13 22:49:27 2011
@@ -18,72 +18,20 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-
-import java.text.NumberFormat;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProtoOrBuilder;
-public class ContainerIdPBImpl extends ProtoBase<ContainerIdProto> implements ContainerId {
+public class ContainerIdPBImpl extends ContainerId {
ContainerIdProto proto = ContainerIdProto.getDefaultInstance();
ContainerIdProto.Builder builder = null;
boolean viaProto = false;
- private ApplicationId applicationId = null;
- private ApplicationAttemptId appAttemptId = null;
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(4);
- }
-
- protected static final NumberFormat counterFormat = NumberFormat.getInstance();
- static {
- counterFormat.setGroupingUsed(false);
- counterFormat.setMinimumIntegerDigits(6);
- }
-
- // TODO: Why thread local?
- // ^ NumberFormat instances are not threadsafe
- private static final ThreadLocal<NumberFormat> appIdFormat = new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(4);
- return fmt;
- }
- };
+ private ApplicationAttemptId applicationAttemptId = null;
- // TODO: fail the app submission if attempts are more than 10 or something
- private static final ThreadLocal<NumberFormat> appAttemptIdFormat = new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(2);
- return fmt;
- }
- };
- // TODO: Why thread local?
- // ^ NumberFormat instances are not threadsafe
- private static final ThreadLocal<NumberFormat> containerIdFormat = new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(6);
- return fmt;
- }
- };
-
public ContainerIdPBImpl() {
builder = ContainerIdProto.newBuilder();
}
@@ -93,23 +41,22 @@ public class ContainerIdPBImpl extends P
viaProto = true;
}
- public ContainerIdProto getProto() {
+ public synchronized ContainerIdProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void mergeLocalToBuilder() {
- if (this.applicationId != null && !((ApplicationIdPBImpl)applicationId).getProto().equals(builder.getAppId())) {
- builder.setAppId(convertToProtoFormat(this.applicationId));
- }
- if (this.appAttemptId != null && !((ApplicationAttemptIdPBImpl)appAttemptId).getProto().equals(builder.getAppAttemptId())) {
- builder.setAppAttemptId(convertToProtoFormat(this.appAttemptId));
+ private synchronized void mergeLocalToBuilder() {
+ if (this.applicationAttemptId != null && !
+ ((ApplicationAttemptIdPBImpl)applicationAttemptId).getProto().equals(
+ builder.getAppAttemptId())) {
+ builder.setAppAttemptId(convertToProtoFormat(this.applicationAttemptId));
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
@@ -117,7 +64,7 @@ public class ContainerIdPBImpl extends P
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerIdProto.newBuilder(proto);
}
@@ -126,109 +73,46 @@ public class ContainerIdPBImpl extends P
@Override
- public int getId() {
+ public synchronized int getId() {
ContainerIdProtoOrBuilder p = viaProto ? proto : builder;
return (p.getId());
}
@Override
- public void setId(int id) {
+ public synchronized void setId(int id) {
maybeInitBuilder();
builder.setId((id));
}
- @Override
- public ApplicationId getAppId() {
- ContainerIdProtoOrBuilder p = viaProto ? proto : builder;
- if (this.applicationId != null) {
- return this.applicationId;
- }
- if (!p.hasAppId()) {
- return null;
- }
- this.applicationId = convertFromProtoFormat(p.getAppId());
- return this.applicationId;
- }
+
@Override
- public ApplicationAttemptId getAppAttemptId() {
+ public synchronized ApplicationAttemptId getApplicationAttemptId() {
ContainerIdProtoOrBuilder p = viaProto ? proto : builder;
- if (this.appAttemptId != null) {
- return this.appAttemptId;
+ if (this.applicationAttemptId != null) {
+ return this.applicationAttemptId;
}
if (!p.hasAppAttemptId()) {
return null;
}
- this.appAttemptId = convertFromProtoFormat(p.getAppAttemptId());
- return this.appAttemptId;
- }
-
- @Override
- public void setAppId(ApplicationId appId) {
- maybeInitBuilder();
- if (appId == null)
- builder.clearAppId();
- this.applicationId = appId;
+ this.applicationAttemptId = convertFromProtoFormat(p.getAppAttemptId());
+ return this.applicationAttemptId;
}
@Override
- public void setAppAttemptId(ApplicationAttemptId atId) {
+ public synchronized void setApplicationAttemptId(ApplicationAttemptId atId) {
maybeInitBuilder();
if (atId == null)
builder.clearAppAttemptId();
- this.appAttemptId = atId;
+ this.applicationAttemptId = atId;
}
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
+ private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+ ApplicationAttemptIdProto p) {
return new ApplicationAttemptIdPBImpl(p);
}
- private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) {
+ private ApplicationAttemptIdProto convertToProtoFormat(
+ ApplicationAttemptId t) {
return ((ApplicationAttemptIdPBImpl)t).getProto();
}
-
- private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
- return new ApplicationIdPBImpl(p);
- }
-
- private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
- return ((ApplicationIdPBImpl)t).getProto();
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null) {
- return false;
- }
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public int compareTo(ContainerId other) {
- if (this.getAppAttemptId().compareTo(other.getAppAttemptId()) == 0) {
- return this.getId() - other.getId();
- } else {
- return this.getAppAttemptId().compareTo(other.getAppAttemptId());
- }
-
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- ApplicationId appId = getAppId();
- sb.append("container_").append(appId.getClusterTimestamp()).append("_");
- sb.append(appIdFormat.get().format(appId.getId())).append("_");
- sb.append(appAttemptIdFormat.get().format(getAppAttemptId().
- getAttemptId())).append("_");
- sb.append(containerIdFormat.get().format(getId()));
- return sb.toString();
- }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java Tue Sep 13 22:49:27 2011
@@ -41,8 +41,11 @@ import org.apache.hadoop.yarn.proto.Yarn
-public class ContainerLaunchContextPBImpl extends ProtoBase<ContainerLaunchContextProto> implements ContainerLaunchContext {
- ContainerLaunchContextProto proto = ContainerLaunchContextProto.getDefaultInstance();
+public class ContainerLaunchContextPBImpl
+extends ProtoBase<ContainerLaunchContextProto>
+implements ContainerLaunchContext {
+ ContainerLaunchContextProto proto =
+ ContainerLaunchContextProto.getDefaultInstance();
ContainerLaunchContextProto.Builder builder = null;
boolean viaProto = false;
@@ -72,10 +75,14 @@ public class ContainerLaunchContextPBImp
}
private void mergeLocalToBuilder() {
- if (this.containerId != null && !((ContainerIdPBImpl)containerId).getProto().equals(builder.getContainerId())) {
+ if (this.containerId != null &&
+ !((ContainerIdPBImpl)containerId).getProto().equals(
+ builder.getContainerId())) {
builder.setContainerId(convertToProtoFormat(this.containerId));
}
- if (this.resource != null && !((ResourcePBImpl)this.resource).getProto().equals(builder.getResource())) {
+ if (this.resource != null &&
+ !((ResourcePBImpl)this.resource).getProto().equals(
+ builder.getResource())) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.localResources != null) {
@@ -131,22 +138,13 @@ public class ContainerLaunchContextPBImp
builder.clearResource();
this.resource = resource;
}
+
@Override
- public List<String> getCommandList() {
+ public List<String> getCommands() {
initCommands();
return this.commands;
}
- @Override
- public String getCommand(int index) {
- initCommands();
- return this.commands.get(index);
- }
- @Override
- public int getCommandCount() {
- initCommands();
- return this.commands.size();
- }
-
+
private void initCommands() {
if (this.commands != null) {
return;
@@ -161,11 +159,12 @@ public class ContainerLaunchContextPBImp
}
@Override
- public void addAllCommands(final List<String> command) {
- if (command == null)
+ public void setCommands(final List<String> commands) {
+ if (commands == null)
return;
initCommands();
- this.commands.addAll(command);
+ this.commands.clear();
+ this.commands.addAll(commands);
}
private void addCommandsToProto() {
@@ -175,21 +174,7 @@ public class ContainerLaunchContextPBImp
return;
builder.addAllCommand(this.commands);
}
- @Override
- public void addCommand(String command) {
- initCommands();
- this.commands.add(command);
- }
- @Override
- public void removeCommand(int index) {
- initCommands();
- this.commands.remove(index);
- }
- @Override
- public void clearCommands() {
- initCommands();
- this.commands.clear();
- }
+
@Override
public String getUser() {
ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
@@ -228,17 +213,13 @@ public class ContainerLaunchContextPBImp
builder.clearContainerId();
this.containerId = containerId;
}
+
@Override
- public Map<String, LocalResource> getAllLocalResources() {
+ public Map<String, LocalResource> getLocalResources() {
initLocalResources();
return this.localResources;
}
- @Override
- public LocalResource getLocalResource(String key) {
- initLocalResources();
- return this.localResources.get(key);
- }
-
+
private void initLocalResources() {
if (this.localResources != null) {
return;
@@ -253,10 +234,12 @@ public class ContainerLaunchContextPBImp
}
@Override
- public void addAllLocalResources(final Map<String, LocalResource> localResources) {
+ public void setLocalResources(
+ final Map<String, LocalResource> localResources) {
if (localResources == null)
return;
initLocalResources();
+ this.localResources.clear();
this.localResources.putAll(localResources);
}
@@ -265,7 +248,8 @@ public class ContainerLaunchContextPBImp
builder.clearLocalResources();
if (localResources == null)
return;
- Iterable<StringLocalResourceMapProto> iterable = new Iterable<StringLocalResourceMapProto>() {
+ Iterable<StringLocalResourceMapProto> iterable =
+ new Iterable<StringLocalResourceMapProto>() {
@Override
public Iterator<StringLocalResourceMapProto> iterator() {
@@ -281,7 +265,8 @@ public class ContainerLaunchContextPBImp
@Override
public StringLocalResourceMapProto next() {
String key = keyIter.next();
- return StringLocalResourceMapProto.newBuilder().setKey(key).setValue(convertToProtoFormat(localResources.get(key))).build();
+ return StringLocalResourceMapProto.newBuilder().setKey(key).
+ setValue(convertToProtoFormat(localResources.get(key))).build();
}
@Override
@@ -293,21 +278,7 @@ public class ContainerLaunchContextPBImp
};
builder.addAllLocalResources(iterable);
}
- @Override
- public void setLocalResource(String key, LocalResource val) {
- initLocalResources();
- this.localResources.put(key, val);
- }
- @Override
- public void removeLocalResource(String key) {
- initLocalResources();
- this.localResources.remove(key);
- }
- @Override
- public void clearLocalResources() {
- initLocalResources();
- this.localResources.clear();
- }
+
@Override
public ByteBuffer getContainerTokens() {
ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
@@ -328,16 +299,12 @@ public class ContainerLaunchContextPBImp
builder.clearContainerTokens();
this.containerTokens = containerTokens;
}
+
@Override
- public Map<String, ByteBuffer> getAllServiceData() {
+ public Map<String, ByteBuffer> getServiceData() {
initServiceData();
return this.serviceData;
}
- @Override
- public ByteBuffer getServiceData(String key) {
- initServiceData();
- return this.serviceData.get(key);
- }
private void initServiceData() {
if (this.serviceData != null) {
@@ -353,7 +320,7 @@ public class ContainerLaunchContextPBImp
}
@Override
- public void addAllServiceData(final Map<String, ByteBuffer> serviceData) {
+ public void setServiceData(final Map<String, ByteBuffer> serviceData) {
if (serviceData == null)
return;
initServiceData();
@@ -365,7 +332,8 @@ public class ContainerLaunchContextPBImp
builder.clearServiceData();
if (serviceData == null)
return;
- Iterable<StringBytesMapProto> iterable = new Iterable<StringBytesMapProto>() {
+ Iterable<StringBytesMapProto> iterable =
+ new Iterable<StringBytesMapProto>() {
@Override
public Iterator<StringBytesMapProto> iterator() {
@@ -381,7 +349,8 @@ public class ContainerLaunchContextPBImp
@Override
public StringBytesMapProto next() {
String key = keyIter.next();
- return StringBytesMapProto.newBuilder().setKey(key).setValue(convertToProtoFormat(serviceData.get(key))).build();
+ return StringBytesMapProto.newBuilder().setKey(key).setValue(
+ convertToProtoFormat(serviceData.get(key))).build();
}
@Override
@@ -393,31 +362,12 @@ public class ContainerLaunchContextPBImp
};
builder.addAllServiceData(iterable);
}
+
@Override
- public void setServiceData(String key, ByteBuffer val) {
- initServiceData();
- this.serviceData.put(key, val);
- }
- @Override
- public void removeServiceData(String key) {
- initServiceData();
- this.serviceData.remove(key);
- }
- @Override
- public void clearServiceData() {
- initServiceData();
- this.serviceData.clear();
- }
- @Override
- public Map<String, String> getAllEnv() {
+ public Map<String, String> getEnv() {
initEnv();
return this.env;
}
- @Override
- public String getEnv(String key) {
- initEnv();
- return this.env.get(key);
- }
private void initEnv() {
if (this.env != null) {
@@ -433,10 +383,11 @@ public class ContainerLaunchContextPBImp
}
@Override
- public void addAllEnv(final Map<String, String> env) {
+ public void setEnv(final Map<String, String> env) {
if (env == null)
return;
initEnv();
+ this.env.clear();
this.env.putAll(env);
}
@@ -445,7 +396,8 @@ public class ContainerLaunchContextPBImp
builder.clearEnv();
if (env == null)
return;
- Iterable<StringStringMapProto> iterable = new Iterable<StringStringMapProto>() {
+ Iterable<StringStringMapProto> iterable =
+ new Iterable<StringStringMapProto>() {
@Override
public Iterator<StringStringMapProto> iterator() {
@@ -461,7 +413,8 @@ public class ContainerLaunchContextPBImp
@Override
public StringStringMapProto next() {
String key = keyIter.next();
- return StringStringMapProto.newBuilder().setKey(key).setValue((env.get(key))).build();
+ return StringStringMapProto.newBuilder().setKey(key).setValue(
+ (env.get(key))).build();
}
@Override
@@ -473,21 +426,6 @@ public class ContainerLaunchContextPBImp
};
builder.addAllEnv(iterable);
}
- @Override
- public void setEnv(String key, String val) {
- initEnv();
- this.env.put(key, val);
- }
- @Override
- public void removeEnv(String key) {
- initEnv();
- this.env.remove(key);
- }
- @Override
- public void clearEnv() {
- initEnv();
- this.env.clear();
- }
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java Tue Sep 13 22:49:27 2011
@@ -31,7 +31,8 @@ import org.apache.hadoop.yarn.util.Proto
-public class ContainerStatusPBImpl extends ProtoBase<ContainerStatusProto> implements ContainerStatus {
+public class ContainerStatusPBImpl extends ProtoBase<ContainerStatusProto>
+implements ContainerStatus {
ContainerStatusProto proto = ContainerStatusProto.getDefaultInstance();
ContainerStatusProto.Builder builder = null;
boolean viaProto = false;
@@ -116,13 +117,13 @@ public class ContainerStatusPBImpl exten
this.containerId = containerId;
}
@Override
- public String getExitStatus() {
+ public int getExitStatus() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
- return (p.getExitStatus());
+ return p.getExitStatus();
}
@Override
- public void setExitStatus(String exitStatus) {
+ public void setExitStatus(int exitStatus) {
maybeInitBuilder();
builder.setExitStatus(exitStatus);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Tue Sep 13 22:49:27 2011
@@ -177,8 +177,8 @@ message ResourceRequestProto {
message AMResponseProto {
optional bool reboot = 1;
optional int32 response_id = 2;
- repeated ContainerProto new_containers = 3;
- repeated ContainerProto finished_containers = 4;
+ repeated ContainerProto allocated_containers = 3;
+ repeated ContainerStatusProto completed_container_statuses = 4;
optional ResourceProto limit = 5;
}
@@ -250,7 +250,7 @@ message ContainerStatusProto {
optional ContainerIdProto container_id = 1;
optional ContainerStateProto state = 2;
optional string diagnostics = 3 [default = "N/A"];
- optional string exit_status = 4 [default = "N/A"];
+ optional int32 exit_status = 4 [default = -1000];
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Sep 13 22:49:27 2011
@@ -27,50 +27,333 @@ public class YarnConfiguration extends C
private static final Splitter ADDR_SPLITTER = Splitter.on(':').trimResults();
private static final Joiner JOINER = Joiner.on("");
- public static final String RM_PREFIX = "yarn.server.resourcemanager.";
-
- public static final String SCHEDULER_ADDRESS = RM_PREFIX
- + "scheduler.address";
-
- public static final String AM_EXPIRY_INTERVAL = RM_PREFIX
- + "application.expiry.interval";
-
- public static final String DEFAULT_SCHEDULER_BIND_ADDRESS = "0.0.0.0:8030";
-
- public static final String APPSMANAGER_ADDRESS = RM_PREFIX
- + "appsManager.address";
-
- public static final String YARN_SECURITY_INFO =
- "yarn.security.info.class.name";
-
- public static final String DEFAULT_APPSMANAGER_BIND_ADDRESS =
- "0.0.0.0:8040";
-
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
private static final String YARN_SITE_XML_FILE = "yarn-site.xml";
- public static final String APPLICATION_MANAGER_PRINCIPAL =
- "yarn.jobmanager.user-name";
-
- public static final String RM_WEBAPP_BIND_ADDRESS = RM_PREFIX
- + "webapp.address";
-
- public static final String DEFAULT_RM_WEBAPP_BIND_ADDRESS = "0.0.0.0:8088";
-
static {
Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE);
Configuration.addDefaultResource(YARN_SITE_XML_FILE);
}
- public static final String RM_SERVER_PRINCIPAL_KEY =
- "yarn.resourcemanager.principal";
-
+ //Configurations
+
+ /** ACL of who can view this application.*/
public static final String APPLICATION_ACL_VIEW_APP =
- "application.acl-view-job";
-
+ "yarn.app.acl.view-job";
+
+ /** ACL of who can modify this application.*/
public static final String APPLICATION_ACL_MODIFY_APP =
- "application.acl-modify-job";
+ "yarn.app.acl.modify-job";
+
+ /**
+ * Security info class This is an internal config set and
+ * read by YARN itself.
+ */
+ public static final String YARN_SECURITY_INFO =
+ "yarn.security.info.class";
+
+ /** Delay before deleting resource to ease debugging of NM issues */
+ public static final String DEBUG_NM_DELETE_DELAY_SEC =
+ YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
+
+ ////////////////////////////////
+ // IPC Configs
+ ////////////////////////////////
+ public static final String IPC_PREFIX = "yarn.ipc.";
+
+ /** Factory to create client IPC classes.*/
+ public static final String IPC_CLIENT_FACTORY =
+ IPC_PREFIX + "client.factory.class";
+
+ /** Type of serialization to use.*/
+ public static final String IPC_SERIALIZER_TYPE =
+ IPC_PREFIX + "serializer.type";
+ public static final String DEFAULT_IPC_SERIALIZER_TYPE = "protocolbuffers";
+
+ /** Factory to create server IPC classes.*/
+ public static final String IPC_SERVER_FACTORY =
+ IPC_PREFIX + "server.factory.class";
+
+ /** Factory to create IPC exceptions.*/
+ public static final String IPC_EXCEPTION_FACTORY =
+ IPC_PREFIX + "exception.factory.class";
+
+ /** Factory to create serializeable records.*/
+ public static final String IPC_RECORD_FACTORY =
+ IPC_PREFIX + "record.factory.class";
+
+ /** RPC class implementation*/
+ public static final String IPC_RPC_IMPL =
+ IPC_PREFIX + "rpc.class";
+ public static final String DEFAULT_IPC_RPC_IMPL =
+ "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC";
+
+ ////////////////////////////////
+ // Resource Manager Configs
+ ////////////////////////////////
+ public static final String RM_PREFIX = "yarn.resourcemanager.";
+
+ /** The address of the applications manager interface in the RM.*/
+ public static final String RM_ADDRESS =
+ RM_PREFIX + "address";
+ public static final String DEFAULT_RM_ADDRESS =
+ "0.0.0.0:8040";
+
+ /** The number of threads used to handle applications manager requests.*/
+ public static final String RM_CLIENT_THREAD_COUNT =
+ RM_PREFIX + "client.thread-count";
+ public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 10;
+
+ /** The expiry interval for application master reporting.*/
+ public static final String RM_AM_EXPIRY_INTERVAL_MS =
+ RM_PREFIX + "am.liveness-monitor.expiry-interval-ms";
+ public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000;
+
+ /** The Kerberos principal for the resource manager.*/
+ public static final String RM_PRINCIPAL =
+ RM_PREFIX + "principal";
+
+ /** The address of the scheduler interface.*/
+ public static final String RM_SCHEDULER_ADDRESS =
+ RM_PREFIX + "scheduler.address";
+ public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:8030";
+
+ /** Number of threads to handle scheduler interface.*/
+ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
+ RM_PREFIX + "scheduler.client.thread-count";
+ public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 10;
+
+ /** The address of the RM web application.*/
+ public static final String RM_WEBAPP_ADDRESS =
+ RM_PREFIX + "webapp.address";
+ public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:8088";
+
+ public static final String RM_RESOURCE_TRACKER_ADDRESS =
+ RM_PREFIX + "resource-tracker.address";
+ public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
+ "0.0.0.0:8025";
+
+ /** Are RM acls enabled.*/
+ public static final String RM_ACL_ENABLE =
+ RM_PREFIX + "acl.enable";
+ public static final boolean DEFAULT_RM_ACL_ENABLE = false;
+
+ /** ACL of who can be admin of RM.*/
+ public static final String RM_ADMIN_ACL =
+ RM_PREFIX + "admin.acl";
+ public static final String DEFAULT_RM_ADMIN_ACL = "*";
+
+ /** The address of the RM admin interface.*/
+ public static final String RM_ADMIN_ADDRESS =
+ RM_PREFIX + "admin.address";
+ public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:8141";
+
+ /**Number of threads used to handle RM admin interface.*/
+ public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
+ RM_PREFIX + "admin.client.thread-count";
+ public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1;
+
+ /** How often should the RM check that the AM is still alive.*/
+ public static final String RM_AM_LIVENESS_MONITOR_INTERVAL_MS =
+ RM_PREFIX + "amliveliness-monitor.interval-ms";
+ public static final int DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS = 1000;
+
+ /** The maximum number of application master retries.*/
+ public static final String RM_AM_MAX_RETRIES =
+ RM_PREFIX + "am.max-retries";
+ public static final int DEFAULT_RM_AM_MAX_RETRIES = 1;
+
+ /** How often to check that containers are still alive. */
+ public static final String RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS =
+ RM_PREFIX + "container.liveness-monitor.interval-ms";
+ public static final int DEFAULT_RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS =
+ 600000;
+
+ /** The keytab for the resource manager.*/
+ public static final String RM_KEYTAB =
+ RM_PREFIX + "keytab";
+
+ /** How long to wait until a node manager is considered dead.*/
+ public static final String RM_NM_EXPIRY_INTERVAL_MS =
+ RM_PREFIX + "nm.liveness-monitor.expiry-interval-ms";
+ public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000;
+
+ /** How often to check that node managers are still alive.*/
+ public static final String RM_NM_LIVENESS_MONITOR_INTERVAL_MS =
+ RM_PREFIX + "nm.liveness-monitor.interval-ms";
+ public static final int DEFAULT_RM_NM_LIVENESS_MONITOR_INTERVAL_MS = 1000;
+
+ /** Path to file with nodes to include.*/
+ public static final String RM_NODES_INCLUDE_FILE_PATH =
+ RM_PREFIX + "nodes.include-path";
+ public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = "";
+
+ /** Path to file with nodes to exclude.*/
+ public static final String RM_NODES_EXCLUDE_FILE_PATH =
+ RM_PREFIX + "nodes.exclude-path";
+ public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = "";
+
+ /** Number of threads to handle resource tracker calls.*/
+ public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT =
+ RM_PREFIX + "resource-tracker.client.thread-count";
+ public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 10;
+
+ /** The class to use as the resource scheduler.*/
+ public static final String RM_SCHEDULER =
+ RM_PREFIX + "scheduler.class";
+
+ /** The class to use as the persistent store.*/
+ public static final String RM_STORE = RM_PREFIX + "store.class";
+
+ /** The address of the zookeeper instance to use with ZK store.*/
+ public static final String RM_ZK_STORE_ADDRESS =
+ RM_PREFIX + "zookeeper-store.address";
+
+ /** The zookeeper session timeout for the zookeeper store.*/
+ public static final String RM_ZK_STORE_TIMEOUT_MS =
+ RM_PREFIX + "zookeeper-store.session.timeout-ms";
+ public static final int DEFAULT_RM_ZK_STORE_TIMEOUT_MS = 60000;
+
+ /** The maximum number of completed applications RM keeps. */
+ public static final String RM_MAX_COMPLETED_APPLICATIONS =
+ RM_PREFIX + "max-completed-applications";
+ public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000;
+
+ ////////////////////////////////
+ // Node Manager Configs
+ ////////////////////////////////
+
+ /** Prefix for all node manager configs.*/
+ public static final String NM_PREFIX = "yarn.nodemanager.";
+
+ /** address of node manager IPC.*/
+ public static final String NM_ADDRESS = NM_PREFIX + "address";
+ public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:45454";
+
+ /** who will execute(launch) the containers.*/
+ public static final String NM_CONTAINER_EXECUTOR =
+ NM_PREFIX + "container-executor.class";
+
+ /** Number of threads container manager uses.*/
+ public static final String NM_CONTAINER_MGR_THREAD_COUNT =
+ NM_PREFIX + "container-manager.thread-count";
+ public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 5;
+
+ /** Number of threads used in cleanup.*/
+ public static final String NM_DELETE_THREAD_COUNT =
+ NM_PREFIX + "delete.thread-count";
+ public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4;
+
+ // TODO: Should this instead be dictated by RM?
+ /** Heartbeat interval to RM*/
+ public static final String NM_TO_RM_HEARTBEAT_INTERVAL_MS =
+ NM_PREFIX + "heartbeat.interval-ms";
+ public static final int DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000;
+
+ /** Keytab for NM.*/
+ public static final String NM_KEYTAB = NM_PREFIX + "keytab";
+
+ /**List of directories to store localized files in.*/
+ public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs";
+ public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
+
+ /** Address where the localizer IPC is.*/
+ public static final String NM_LOCALIZER_ADDRESS =
+ NM_PREFIX + "localizer.address";
+ public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:4344";
+
+ /** Interval in between cache cleanups.*/
+ public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
+ NM_PREFIX + "localizer.cache.cleanup.interval-ms";
+ public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
+ 10 * 60 * 1000;
+
+ /** Target size of localizer cache in MB, per local directory.*/
+ public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
+ NM_PREFIX + "localizer.cache.target-size-mb";
+ public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;
+
+ /** Number of threads to handle localization requests.*/
+ public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT =
+ NM_PREFIX + "localizer.client.thread-count";
+ public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5;
+
+ /** Number of threads to use for localization fetching.*/
+ public static final String NM_LOCALIZER_FETCH_THREAD_COUNT =
+ NM_PREFIX + "localizer.fetch.thread-count";
+ public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4;
+
+ /** Where to store container logs.*/
+ public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
+ public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
+
+ /** Where to aggregate logs to.*/
+ public static final String NM_REMOTE_APP_LOG_DIR =
+ NM_PREFIX + "remote-app-log-dir";
+ public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs";
+
+ /** Amount of memory in GB that can be allocated for containers.*/
+ public static final String NM_VMEM_GB = NM_PREFIX + "resource.memory-gb";
+ public static final int DEFAULT_NM_VMEM_GB = 8;
+
+ /** NM Webapp address.**/
+ public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
+ public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:9999";
+
+ /** How often to monitor containers.*/
+ public final static String NM_CONTAINER_MON_INTERVAL_MS =
+ NM_PREFIX + "container-monitor.interval-ms";
+ public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000;
+
+ /** Class that calculates containers current resource utilization.*/
+ public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR =
+ NM_PREFIX + "container-monitor.resource-calculator.class";
+
+ /** Amount of physical ram to reserve for other applications, -1 disables.*/
+ public static final String NM_RESERVED_MEMORY_MB =
+ NM_PREFIX + "reserved.memory-mb";
+
+ /** Frequency of running node health script.*/
+ public static final String NM_HEALTH_CHECK_INTERVAL_MS =
+ NM_PREFIX + "health-checker.interval-ms";
+ public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000;
+
+ /** Script time out period.*/
+ public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS =
+ NM_PREFIX + "health-checker.script.timeout-ms";
+ public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS =
+ 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS;
+
+ /** The health check script to run.*/
+ public static final String NM_HEALTH_CHECK_SCRIPT_PATH =
+ NM_PREFIX + "health-checker.script.path";
+
+ /** The arguments to pass to the health check script.*/
+ public static final String NM_HEALTH_CHECK_SCRIPT_OPTS =
+ NM_PREFIX + "health-checker.script.opts";
+
+ /** The path to the Linux container executor.*/
+ public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
+ NM_PREFIX + "linux-container-executor.path";
+
+ /** T-file compression types used to compress aggregated logs.*/
+ public static final String NM_LOG_AGG_COMPRESSION_TYPE =
+ NM_PREFIX + "log-aggregation.compression-type";
+ public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none";
+
+ /** The kerberos principal for the node manager.*/
+ public static final String NM_PRINCIPAL =
+ NM_PREFIX + "principal";
+
+ public static final String NM_AUX_SERVICES =
+ NM_PREFIX + "aux-services";
+
+ public static final String NM_AUX_SERVICE_FMT =
+ NM_PREFIX + "aux-services.%s.class";
+ public static final int INVALID_CONTAINER_EXIT_STATUS = -1000;
+ public static final int ABORTED_CONTAINER_EXIT_STATUS = -100;
+
public YarnConfiguration() {
super();
}
@@ -83,13 +366,13 @@ public class YarnConfiguration extends C
}
public static String getRMWebAppURL(Configuration conf) {
- String addr = conf.get(RM_WEBAPP_BIND_ADDRESS,
- DEFAULT_RM_WEBAPP_BIND_ADDRESS);
+ String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
it.next(); // ignore the bind host
String port = it.next();
// Use apps manager address to figure out the host for webapp
- addr = conf.get(APPSMANAGER_ADDRESS, DEFAULT_APPSMANAGER_BIND_ADDRESS);
+ addr = conf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS);
String host = ADDR_SPLITTER.split(addr).iterator().next();
return JOINER.join("http://", host, ":", port, "/");
}