You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/09/08 03:39:23 UTC
svn commit: r1166495 [4/6] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/mai...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Thu Sep 8 01:39:07 2011
@@ -22,68 +22,319 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p><code>ApplicationSubmissionContext</code> represents the all of the
+ * information needed by the <code>ResourceManager</code> to launch
+ * the <code>ApplicationMaster</code> for an application.</p>
+ *
+ * <p>It includes details such as:
+ * <ul>
+ * <li>{@link ApplicationId} of the application.</li>
+ * <li>
+ * {@link Resource} necessary to run the <code>ApplicationMaster</code>.
+ * </li>
+ * <li>Application user.</li>
+ * <li>Application name.</li>
+ * <li>{@link Priority} of the application.</li>
+ * <li>Security tokens (if security is enabled).</li>
+ * <li>
+ * {@link LocalResource} necessary for running the
+ * <code>ApplicationMaster</code> container such
+ * as binaries, jar, shared-objects, side-files etc.
+ * </li>
+ * <li>
+ * Environment variables for the launched <code>ApplicationMaster</code>
+ * process.
+ * </li>
+ * <li>Command to launch the <code>ApplicationMaster</code>.</li>
+ * </ul>
+ * </p>
+ *
+ * @see ClientRMProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)
+ */
+@Public
+@Stable
public interface ApplicationSubmissionContext {
- public abstract ApplicationId getApplicationId();
- public abstract String getApplicationName();
- public abstract Resource getMasterCapability();
-
- public abstract Map<String, URL> getAllResources();
- public abstract URL getResource(String key);
-
- public abstract Map<String, LocalResource> getAllResourcesTodo();
- public abstract LocalResource getResourceTodo(String key);
-
- public abstract List<String> getFsTokenList();
- public abstract String getFsToken(int index);
- public abstract int getFsTokenCount();
-
- public abstract ByteBuffer getFsTokensTodo();
-
- public abstract Map<String, String> getAllEnvironment();
- public abstract String getEnvironment(String key);
-
- public abstract List<String> getCommandList();
- public abstract String getCommand(int index);
- public abstract int getCommandCount();
-
- public abstract String getQueue();
- public abstract Priority getPriority();
- public abstract String getUser();
+ /**
+ * Get the <code>ApplicationId</code> of the submitted application.
+ * @return <code>ApplicationId</code> of the submitted application
+ */
+ @Public
+ @Stable
+ public ApplicationId getApplicationId();
+
+ /**
+ * Set the <code>ApplicationId</code> of the submitted application.
+ * @param appplicationId <code>ApplicationId</code> of the submitted
+ * application
+ */
+ @Public
+ @Stable
+ public void setApplicationId(ApplicationId appplicationId);
-
-
- public abstract void setApplicationId(ApplicationId appplicationId);
- public abstract void setApplicationName(String applicationName);
- public abstract void setMasterCapability(Resource masterCapability);
-
- public abstract void addAllResources(Map<String, URL> resources);
- public abstract void setResource(String key, URL url);
- public abstract void removeResource(String key);
- public abstract void clearResources();
-
- public abstract void addAllResourcesTodo(Map<String, LocalResource> resourcesTodo);
- public abstract void setResourceTodo(String key, LocalResource localResource);
- public abstract void removeResourceTodo(String key);
- public abstract void clearResourcesTodo();
-
- public abstract void addAllFsTokens(List<String> fsTokens);
- public abstract void addFsToken(String fsToken);
- public abstract void removeFsToken(int index);
- public abstract void clearFsTokens();
-
- public abstract void setFsTokensTodo(ByteBuffer fsTokensTodo);
-
- public abstract void addAllEnvironment(Map<String, String> environment);
- public abstract void setEnvironment(String key, String env);
- public abstract void removeEnvironment(String key);
- public abstract void clearEnvironment();
-
- public abstract void addAllCommands(List<String> commands);
- public abstract void addCommand(String command);
- public abstract void removeCommand(int index);
- public abstract void clearCommands();
-
- public abstract void setQueue(String queue);
- public abstract void setPriority(Priority priority);
- public abstract void setUser(String user);
+ /**
+ * Get the application <em>name</em>.
+ * @return application name
+ */
+ @Public
+ @Stable
+ public String getApplicationName();
+
+ /**
+ * Set the application <em>name</em>.
+ * @param applicationName application name
+ */
+ @Public
+ @Stable
+ public void setApplicationName(String applicationName);
+
+ /**
+ * Get the <em>queue</em> to which the application is being submitted.
+ * @return <em>queue</em> to which the application is being submitted
+ */
+ @Public
+ @Stable
+ public String getQueue();
+
+ /**
+ * Set the <em>queue</em> to which the application is being submitted
+ * @param queue <em>queue</em> to which the application is being submitted
+ */
+ @Public
+ @Stable
+ public void setQueue(String queue);
+
+ /**
+ * Get the <code>Priority</code> of the application.
+ * @return <code>Priority</code> of the application
+ */
+ @Public
+ @Stable
+ public Priority getPriority();
+
+ /**
+ * Set the <code>Priority</code> of the application.
+ * @param priority <code>Priority</code> of the application
+ */
+ @Public
+ @Stable
+ public void setPriority(Priority priority);
+
+ /**
+ * Get the <em>user</em> submitting the application.
+ * @return <em>user</em> submitting the application
+ */
+ @Public
+ @Stable
+ public String getUser();
+
+ /**
+ * Set the <em>user</em> submitting the application.
+ * @param user <em>user</em> submitting the application
+ */
+ @Public
+ @Stable
+ public void setUser(String user);
+
+ /**
+ * Get the <code>Resource</code> required to run the
+ * <code>ApplicationMaster</code>.
+ * @return <code>Resource</code> required to run the
+ * <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public Resource getMasterCapability();
+
+ /**
+ * Set <code>Resource</code> required to run the
+ * <code>ApplicationMaster</code>.
+ * @param masterCapability <code>Resource</code> required to run the
+ * <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public void setMasterCapability(Resource masterCapability);
+
+ @Private
+ @Unstable
+ public Map<String, URL> getAllResources();
+
+ @Private
+ @Unstable
+ public URL getResource(String key);
+
+ @Private
+ @Unstable
+ public void addAllResources(Map<String, URL> resources);
+
+ @Private
+ @Unstable
+ public void setResource(String key, URL url);
+
+ @Private
+ @Unstable
+ public void removeResource(String key);
+
+ @Private
+ @Unstable
+ public void clearResources();
+
+ /**
+ * Get all the <code>LocalResource</code> required to run the
+ * <code>ApplicationMaster</code>.
+ * @return <code>LocalResource</code> required to run the
+ * <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public Map<String, LocalResource> getAllResourcesTodo();
+
+ @Private
+ @Unstable
+ public LocalResource getResourceTodo(String key);
+
+ /**
+ * Add all the <code>LocalResource</code> required to run the
+ * <code>ApplicationMaster</code>.
+ * @param resources all <code>LocalResource</code> required to run the
+ * <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public void addAllResourcesTodo(Map<String, LocalResource> resources);
+
+ @Private
+ @Unstable
+ public void setResourceTodo(String key, LocalResource localResource);
+
+ @Private
+ @Unstable
+ public void removeResourceTodo(String key);
+
+ @Private
+ @Unstable
+ public void clearResourcesTodo();
+
+ @Private
+ @Unstable
+ public List<String> getFsTokenList();
+
+ @Private
+ @Unstable
+ public String getFsToken(int index);
+
+ @Private
+ @Unstable
+ public int getFsTokenCount();
+
+ @Private
+ @Unstable
+ public void addAllFsTokens(List<String> fsTokens);
+
+ @Private
+ @Unstable
+ public void addFsToken(String fsToken);
+
+ @Private
+ @Unstable
+ public void removeFsToken(int index);
+
+ @Private
+ @Unstable
+ public void clearFsTokens();
+
+ /**
+ * Get <em>file-system tokens</em> for the <code>ApplicationMaster</code>.
+ * @return file-system tokens for the <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public ByteBuffer getFsTokensTodo();
+
+ /**
+ * Set <em>file-system tokens</em> for the <code>ApplicationMaster</code>.
+ * @param fsTokens file-system tokens for the <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public void setFsTokensTodo(ByteBuffer fsTokens);
+
+ /**
+ * Get the <em>environment variables</em> for the
+ * <code>ApplicationMaster</code>.
+ * @return environment variables for the <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public Map<String, String> getAllEnvironment();
+
+ @Private
+ @Unstable
+ public String getEnvironment(String key);
+
+ /**
+ * Add all of the <em>environment variables</em> for the
+ * <code>ApplicationMaster</code>.
+ * @param environment environment variables for the
+ * <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public void addAllEnvironment(Map<String, String> environment);
+
+ @Private
+ @Unstable
+ public void setEnvironment(String key, String env);
+
+ @Private
+ @Unstable
+ public void removeEnvironment(String key);
+
+ @Private
+ @Unstable
+ public void clearEnvironment();
+
+ /**
+ * Get the <em>commands</em> to launch the <code>ApplicationMaster</code>.
+ * @return commands to launch the <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public List<String> getCommandList();
+
+ @Private
+ @Unstable
+ public String getCommand(int index);
+
+ @Private
+ @Unstable
+ public int getCommandCount();
+
+ /**
+ * Add all of the <em>commands</em> to launch the
+ * <code>ApplicationMaster</code>.
+ * @param commands commands to launch the <code>ApplicationMaster</code>
+ */
+ @Public
+ @Stable
+ public void addAllCommands(List<String> commands);
+
+ @Private
+ @Unstable
+ public void addCommand(String command);
+
+ @Private
+ @Unstable
+ public void removeCommand(int index);
+
+ @Private
+ @Unstable
+ public void clearCommands();
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java Thu Sep 8 01:39:07 2011
@@ -22,50 +22,233 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ContainerManager;
+
+/**
+ * <p><code>ContainerLaunchContext</code> represents the all of the information
+ * needed by the <code>NodeManager</code> to launch a container.</p>
+ *
+ * <p>It includes details such as:
+ * <ul>
+ * <li>{@link ContainerId} of the container.</li>
+ * <li>{@link Resource} allocated to the container.</li>
+ * <li>User to whom the container is allocated.</li>
+ * <li>Security tokens (if security is enabled).</li>
+ * <li>
+ * {@link LocalResource} necessary for running the container such
+ * as binaries, jar, shared-objects, side-files etc.
+ * </li>
+ * <li>Optional, application-specific binary service data.</li>
+ * <li>Environment variables for the launched process.</li>
+ * <li>Command to launch the container.</li>
+ * </ul>
+ * </p>
+ *
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ */
+@Public
+@Stable
public interface ContainerLaunchContext {
+ /**
+ * Get <code>ContainerId</code> of container to be launched.
+ * @return <code>ContainerId</code> of container to be launched
+ */
+ @Public
+ @Stable
ContainerId getContainerId();
+
+ /**
+ * Set <code>ContainerId</code> of container to be launched.
+ * @param containerId et <code>ContainerId</code> of container to be launched
+ */
+ @Public
+ @Stable
+ void setContainerId(ContainerId containerId);
+
+ /**
+ * Get the <em>user</em> to whom the container has been allocated.
+ * @return the <em>user</em> to whom the container has been allocated
+ */
+ @Public
+ @Stable
String getUser();
- Resource getResource();
-
- Map<String, LocalResource> getAllLocalResources();
- LocalResource getLocalResource(String key);
-
- ByteBuffer getContainerTokens();
-
- Map<String, ByteBuffer> getAllServiceData();
- ByteBuffer getServiceData(String key);
-
- Map<String, String> getAllEnv();
- String getEnv(String key);
-
- List<String> getCommandList();
- String getCommand(int index);
- int getCommandCount();
-
- void setContainerId(ContainerId containerId);
+ /**
+ * Set the <em>user</em> to whom the container has been allocated
+ * @param user <em>user</em> to whom the container has been allocated
+ */
+ @Public
+ @Stable
void setUser(String user);
+
+ /**
+ * Get the <code>Resource</code> allocated to the container by the
+ * <code>ResourceManager</code>.
+ * @return <code>Resource</code> allocated to the container by the
+ * <code>ResourceManager</code>
+ */
+ @Public
+ @Stable
+ Resource getResource();
+
+ /**
+ * Set the <code>Resource</code> allocated to the container by the
+ * <code>ResourceManager</code>.
+ * @param resource allocated resource
+ */
+ @Public
+ @Stable
void setResource(Resource resource);
+
+ /**
+ * Get security tokens (if security is enabled).
+ * @return security tokens (if security is enabled)
+ */
+ @Public
+ @Stable
+ ByteBuffer getContainerTokens();
+
+ /**
+ * Set security tokens (if security is enabled).
+ * @param containerToken security tokens
+ */
+ @Public
+ @Stable
+ void setContainerTokens(ByteBuffer containerToken);
+
+ /**
+ * Get all <code>LocalResource</code> required by the container.
+ * @return all <code>LocalResource</code> required by the container
+ */
+ @Public
+ @Stable
+ Map<String, LocalResource> getAllLocalResources();
+ @Private
+ @Unstable
+ LocalResource getLocalResource(String key);
+
+ /**
+ * Add all <code>LocalResource</code> required by the container.
+ * @param localResources <code>LocalResource</code> required by the container
+ */
+ @Public
+ @Stable
void addAllLocalResources(Map<String, LocalResource> localResources);
+
+ @Private
+ @Unstable
void setLocalResource(String key, LocalResource value);
+
+ @Private
+ @Unstable
void removeLocalResource(String key);
+
+ @Private
+ @Unstable
void clearLocalResources();
+
+ /**
+ * Get application-specific binary service data.
+ * @return application-specific binary service data
+ */
+ @Public
+ @Stable
+ Map<String, ByteBuffer> getAllServiceData();
- void setContainerTokens(ByteBuffer containerToken);
-
+ @Private
+ @Unstable
+ ByteBuffer getServiceData(String key);
+
+ /**
+ * Add add application-specific binary service data.
+ * @param serviceData application-specific binary service data
+ */
+ @Public
+ @Stable
void addAllServiceData(Map<String, ByteBuffer> serviceData);
+
+ @Private
+ @Unstable
void setServiceData(String key, ByteBuffer value);
+
+ @Private
+ @Unstable
void removeServiceData(String key);
+
+ @Private
+ @Unstable
void clearServiceData();
+
+ /**
+ * Get <em>environment variables</em> for the launched container.
+ * @return <em>environment variables</em> for the launched container
+ */
+ @Public
+ @Stable
+ Map<String, String> getAllEnv();
+
+ @Private
+ @Unstable
+ String getEnv(String key);
+ /**
+ * Add <em>environment variables</em> for the launched container.
+ * @param env <em>environment variables</em> for the launched container
+ */
+ @Public
+ @Stable
void addAllEnv(Map<String, String> env);
+
+ @Private
+ @Unstable
void setEnv(String key, String value);
+
+ @Private
+ @Unstable
void removeEnv(String key);
+
+ @Private
+ @Unstable
void clearEnv();
+
+ /**
+ * Get the list of <em>commands</em> for launching the container.
+ * @return the list of <em>commands</em> for launching the container
+ */
+ @Public
+ @Stable
+ List<String> getCommandList();
+ @Private
+ @Unstable
+ String getCommand(int index);
+
+ @Private
+ @Unstable
+ int getCommandCount();
+
+ /**
+ * Add the list of <em>commands</em> for launching the container.
+ * @param commands the list of <em>commands</em> for launching the container
+ */
+ @Public
+ @Stable
void addAllCommands(List<String> commands);
+
+ @Private
+ @Unstable
void addCommand(String command);
+
+ @Private
+ @Unstable
void removeCommand(int index);
+
+ @Private
+ @Unstable
void clearCommands();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java Thu Sep 8 01:39:07 2011
@@ -18,16 +18,92 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ContainerManager;
+
+/**
+ * <p><code>LocalResource</code> represents a local resource required to
+ * run a container.</p>
+ *
+ * <p>The <code>NodeManager</code> is responsible for localizing the resource
+ * prior to launching the container.</p>
+ *
+ * <p>Applications can specify {@link LocalResourceType} and
+ * {@link LocalResourceVisibility}.</p>
+ *
+ * @see LocalResourceType
+ * @see LocalResourceVisibility
+ * @see ContainerLaunchContext
+ * @see ApplicationSubmissionContext
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ */
+@Public
+@Stable
public interface LocalResource {
- public abstract URL getResource();
- public abstract long getSize();
- public abstract long getTimestamp();
- public abstract LocalResourceType getType();
- public abstract LocalResourceVisibility getVisibility();
-
- public abstract void setResource(URL resource);
- public abstract void setSize(long size);
- public abstract void setTimestamp(long timestamp);
- public abstract void setType(LocalResourceType type);
- public abstract void setVisibility(LocalResourceVisibility visibility);
+ /**
+ * Get the <em>location</em> of the resource to be localized.
+ * @return <em>location</em> of the resource to be localized
+ */
+ public URL getResource();
+
+ /**
+ * Set <em>location</em> of the resource to be localized.
+ * @param resource <em>location</em> of the resource to be localized
+ */
+ public void setResource(URL resource);
+
+ /**
+ * Get the <em>size</em> of the resource to be localized.
+ * @return <em>size</em> of the resource to be localized
+ */
+ public long getSize();
+
+ /**
+ * Set the <em>size</em> of the resource to be localized.
+ * @param size <em>size</em> of the resource to be localized
+ */
+ public void setSize(long size);
+
+ /**
+ * Get the original <em>timestamp</em> of the resource to be localized, used
+ * for verification.
+ * @return <em>timestamp</em> of the resource to be localized
+ */
+ public long getTimestamp();
+
+ /**
+ * Set the <em>timestamp</em> of the resource to be localized, used
+ * for verification.
+ * @param timestamp <em>timestamp</em> of the resource to be localized
+ */
+ public void setTimestamp(long timestamp);
+
+ /**
+ * Get the <code>LocalResourceType</code> of the resource to be localized.
+ * @return <code>LocalResourceType</code> of the resource to be localized
+ */
+ public LocalResourceType getType();
+
+ /**
+ * Set the <code>LocalResourceType</code> of the resource to be localized.
+ * @param type <code>LocalResourceType</code> of the resource to be localized
+ */
+ public void setType(LocalResourceType type);
+
+ /**
+ * Get the <code>LocalResourceVisibility</code> of the resource to be
+ * localized.
+ * @return <code>LocalResourceVisibility</code> of the resource to be
+ * localized
+ */
+ public LocalResourceVisibility getVisibility();
+
+ /**
+ * Set the <code>LocalResourceVisibility</code> of the resource to be
+ * localized.
+ * @param visibility <code>LocalResourceVisibility</code> of the resource to be
+ * localized
+ */
+ public void setVisibility(LocalResourceVisibility visibility);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java Thu Sep 8 01:39:07 2011
@@ -18,6 +18,42 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ContainerManager;
+
+/**
+ * <p><code>LocalResourceType</code> specifies the <em>type</em>
+ * of a resource localized by the <code>NodeManager</code>.</p>
+ *
+ * <p>The <em>type</em> can be one of:
+ * <ul>
+ * <li>
+ * {@link #FILE} - Regular file i.e. uninterpreted bytes.
+ * </li>
+ * <li>
+ * {@link #ARCHIVE} - Archive, which is automatically unarchived by the
+ * <code>NodeManager</code>.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * @see LocalResource
+ * @see ContainerLaunchContext
+ * @see ApplicationSubmissionContext
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ */
+@Public
+@Stable
public enum LocalResourceType {
- ARCHIVE, FILE
+
+ /**
+ * Archive, which is automatically unarchived by the <code>NodeManager</code>.
+ */
+ ARCHIVE,
+
+ /**
+ * Regular file i.e. uninterpreted bytes.
+ */
+ FILE
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java Thu Sep 8 01:39:07 2011
@@ -18,6 +18,48 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ContainerManager;
+
+/**
+ * <p><code>LocalResourceVisibility</code> specifies the <em>visibility</em>
+ * of a resource localized by the <code>NodeManager</code>.</p>
+ *
+ * <p>The <em>visibility</em> can be one of:
+ * <ul>
+ * <li>{@link #PUBLIC} - Shared by all users on the node.</li>
+ * <li>
+ * {@link #PRIVATE} - Shared among all applications of the
+ * <em>same user</em> on the node.
+ * </li>
+ * <li>
+ * {@link #APPLICATION} - Shared only among containers of the
+ * <em>same application</em> on the node.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * @see LocalResource
+ * @see ContainerLaunchContext
+ * @see ApplicationSubmissionContext
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ */
+@Public
+@Stable
public enum LocalResourceVisibility {
- PUBLIC, PRIVATE, APPLICATION
+ /**
+ * Shared by all users on the node.
+ */
+ PUBLIC,
+
+ /**
+ * Shared among all applications of the <em>same user</em> on the node.
+ */
+ PRIVATE,
+
+ /**
+ * Shared only among containers of the <em>same application</em> on the node.
+ */
+ APPLICATION
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeHealthStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeHealthStatus.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeHealthStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeHealthStatus.java Thu Sep 8 01:39:07 2011
@@ -17,17 +17,69 @@
*/
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p><code>NodeHealthStatus</code> is a summary of the health status of the
+ * node.</p>
+ *
+ * <p>It includes information such as:
+ * <ul>
+ * <li>
+ * An indicator of whether the node is healthy, as determined by the
+ * health-check script.
+ * </li>
+ * <li>The previous time at which the health status was reported.</li>
+ * <li>A diagnostic report on the health status.</li>
+ * <li></li>
+ * <li></li>
+ * </ul>
+ * </p>
+ *
+ * @see NodeReport
+ * @see ClientRMProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
+ */
+@Public
+@Stable
public interface NodeHealthStatus {
+ /**
+ * Is the node healthy?
+ * @return <code>true</code> if the node is healthy, else <code>false</code>
+ */
+ @Public
+ @Stable
boolean getIsNodeHealthy();
- String getHealthReport();
-
- long getLastHealthReportTime();
-
+ @Private
+ @Unstable
void setIsNodeHealthy(boolean isNodeHealthy);
+ /**
+ * Get the <em>diagnostic health report</em> of the node.
+ * @return <em>diagnostic health report</em> of the node
+ */
+ @Public
+ @Stable
+ String getHealthReport();
+
+ @Private
+ @Unstable
void setHealthReport(String healthReport);
+ /**
+ * Get the <em>last timestamp</em> at which the health report was received.
+ * @return <em>last timestamp</em> at which the health report was received
+ */
+ @Public
+ @Stable
+ long getLastHealthReportTime();
+
+ @Private
+ @Unstable
void setLastHealthReportTime(long lastHealthReport);
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java Thu Sep 8 01:39:07 2011
@@ -18,19 +18,113 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p><code>NodeReport</code> is a summary of runtime information of a
+ * node in the cluster.</p>
+ *
+ * <p>It includes details such as:
+ * <ul>
+ * <li>{@link NodeId} of the node.</li>
+ * <li>HTTP Tracking URL of the node.</li>
+ * <li>Rack name for the node.</li>
+ * <li>Used {@link Resource} on the node.</li>
+ * <li>Total available {@link Resource} of the node.</li>
+ * <li>Number of running containers on the node.</li>
+ * <li>{@link NodeHealthStatus} of the node.</li>
+ * </ul>
+ * </p>
+ *
+ * @see NodeHealthStatus
+ * @see ClientRMProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
+ */
+@Public
+@Stable
public interface NodeReport {
+ /**
+ * Get the <code>NodeId</code> of the node.
+ * @return <code>NodeId</code> of the node
+ */
NodeId getNodeId();
+
+ @Private
+ @Unstable
void setNodeId(NodeId nodeId);
+
+ /**
+ * Get the <em>http address</em> of the node.
+ * @return <em>http address</em> of the node
+ */
+ @Public
+ @Stable
String getHttpAddress();
+
+ @Private
+ @Unstable
void setHttpAddress(String httpAddress);
+
+ /**
+ * Get the <em>rack name</em> for the node.
+ * @return <em>rack name</em> for the node
+ */
+ @Public
+ @Stable
String getRackName();
+
+ @Private
+ @Unstable
void setRackName(String rackName);
+
+ /**
+ * Get <em>used</em> <code>Resource</code> on the node.
+ * @return <em>used</em> <code>Resource</code> on the node
+ */
+ @Public
+ @Stable
Resource getUsed();
+
+ @Private
+ @Unstable
void setUsed(Resource used);
+
+ /**
+ * Get the <em>total</em> <code>Resource</code> on the node.
+ * @return <em>total</em> <code>Resource</code> on the node
+ */
+ @Public
+ @Stable
Resource getCapability();
+
+ @Private
+ @Unstable
void setCapability(Resource capability);
+
+ /**
+ * Get the <em>number of running containers</em> on the node.
+ * @return <em>number of running containers</em> on the node
+ */
+ @Public
+ @Stable
int getNumContainers();
+
+ @Private
+ @Unstable
void setNumContainers(int numContainers);
+
+ /**
+ * Get the <code>NodeHealthStatus</code> of the node.
+ * @return <code>NodeHealthStatus</code> of the node
+ */
+ @Public
+ @Stable
NodeHealthStatus getNodeHealthStatus();
+
+ @Private
+ @Unstable
void setNodeHealthStatus(NodeHealthStatus nodeHealthStatus);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueACL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueACL.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueACL.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueACL.java Thu Sep 8 01:39:07 2011
@@ -18,8 +18,39 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p><code>QueueACL</code> enumerates the various ACLs for queues.</p>
+ *
+ * <p>The ACLs are one of:
+ * <ul>
+ * <li>{@link #SUBMIT_JOB} - ACL to submit jobs to the queue.</li>
+ * <li>{@link #ADMINISTER_QUEUE} - ACL to administer the queue.</li>
+ * <li>{@link #ADMINISTER_JOBS} - ACL to administer jobs in the queue.</li>
+ * </ul>
+ * </p>
+ *
+ * @see QueueInfo
+ * @see ClientRMProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)
+ */
+@Public
+@Stable
public enum QueueACL {
+ /**
+ * ACL to submit jobs to the queue.
+ */
SUBMIT_JOB,
+
+ /**
+ * ACL to administer the queue.
+ */
ADMINISTER_QUEUE,
+
+ /**
+ * ACL to administer jobs in the queue.
+ */
ADMINISTER_JOBS; // currently unused
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java Thu Sep 8 01:39:07 2011
@@ -20,25 +20,114 @@ package org.apache.hadoop.yarn.api.recor
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p>QueueInfo</p> is a report of the runtime information of the queue.</p>
+ *
+ * <p>It includes information such as:
+ * <ul>
+ * <li>Queue name.</li>
+ * <li>Capacity of the queue.</li>
+ * <li>Maximum capacity of the queue.</li>
+ * <li>Current capacity of the queue.</li>
+ * <li>Child queues.</li>
+ * <li>Running applications.</li>
+ * <li>{@link QueueState} of the queue.</li>
+ * </ul>
+ * </p>
+ *
+ * @see QueueState
+ * @see ClientRMProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)
+ */
+@Public
+@Stable
public interface QueueInfo {
+ /**
+ * Get the <em>name</em> of the queue.
+ * @return <em>name</em> of the queue
+ */
+ @Public
+ @Stable
String getQueueName();
+
+ @Private
+ @Unstable
void setQueueName(String queueName);
+ /**
+ * Get the <em>configured capacity</em> of the queue.
+ * @return <em>configured capacity</em> of the queue
+ */
+ @Public
+ @Stable
float getCapacity();
+
+ @Private
+ @Unstable
void setCapacity(float capacity);
+ /**
+ * Get the <em>maximum capacity</em> of the queue.
+ * @return <em>maximum capacity</em> of the queue
+ */
+ @Public
+ @Stable
float getMaximumCapacity();
+
+ @Private
+ @Unstable
void setMaximumCapacity(float maximumCapacity);
+ /**
+ * Get the <em>current capacity</em> of the queue.
+ * @return <em>current capacity</em> of the queue
+ */
+ @Public
+ @Stable
float getCurrentCapacity();
+
+ @Private
+ @Unstable
void setCurrentCapacity(float currentCapacity);
+ /**
+ * Get the <em>child queues</em> of the queue.
+ * @return <em>child queues</em> of the queue
+ */
+ @Public
+ @Stable
List<QueueInfo> getChildQueues();
+
+ @Private
+ @Unstable
void setChildQueues(List<QueueInfo> childQueues);
+ /**
+ * Get the <em>running applications</em> of the queue.
+ * @return <em>running applications</em> of the queue
+ */
+ @Public
+ @Stable
List<ApplicationReport> getApplications();
+
+ @Private
+ @Unstable
void setApplications(List<ApplicationReport> applications);
+ /**
+ * Get the <code>QueueState</code> of the queue.
+ * @return <code>QueueState</code> of the queue
+ */
+ @Public
+ @Stable
QueueState getQueueState();
+
+ @Private
+ @Unstable
void setQueueState(QueueState queueState);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java Thu Sep 8 01:39:07 2011
@@ -18,10 +18,33 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
/**
- * State of a Queue
+ * <p>State of a Queue.</p>
+ *
+ * <p>A queue is one of:
+ * <ul>
+ * <li>{@link #RUNNING} - normal state.</li>
+ * <li>{@link #STOPPED} - not accepting new application submissions.
+ * </ul>
+ * </p>
+ *
+ * @see QueueInfo
+ * @see ClientRMProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)
*/
+@Public
+@Stable
public enum QueueState {
- STOPPED,
+ /**
+ * Stopped - Not accepting submissions of new applications.
+ */
+ STOPPED,
+
+ /**
+ * Running - normal operation.
+ */
RUNNING
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueUserACLInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueUserACLInfo.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueUserACLInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueUserACLInfo.java Thu Sep 8 01:39:07 2011
@@ -20,10 +20,43 @@ package org.apache.hadoop.yarn.api.recor
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+
+/**
+ * <p><code>QueueUserACLInfo</code> provides information {@link QueueACL} for
+ * the given user.</p>
+ *
+ * @see QueueACL
+ * @see ClientRMProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)
+ */
+@Public
+@Stable
public interface QueueUserACLInfo {
+ /**
+ * Get the <em>queue name</em> of the queue.
+ * @return <em>queue name</em> of the queue
+ */
+ @Public
+ @Stable
String getQueueName();
- void setQueueName(String queueName);
+ @Private
+ @Unstable
+ void setQueueName(String queueName);
+
+ /**
+ * Get the list of <code>QueueACL</code> for the given user.
+ * @return list of <code>QueueACL</code> for the given user
+ */
+ @Public
+ @Stable
List<QueueACL> getUserAcls();
+
+ @Private
+ @Unstable
void setUserAcls(List<QueueACL> acls);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java Thu Sep 8 01:39:07 2011
@@ -29,8 +29,6 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProtoOrBuilder;
-import org.mortbay.log.Log;
-
public class ContainerIdPBImpl extends ProtoBase<ContainerIdProto> implements ContainerId {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Thu Sep 8 01:39:07 2011
@@ -143,6 +143,7 @@ message StartContainerRequestProto {
}
message StartContainerResponseProto {
+ repeated StringBytesMapProto service_response = 1;
}
message StopContainerRequestProto {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java Thu Sep 8 01:39:07 2011
@@ -317,15 +317,15 @@ public class ProtoOverHadoopRpcEngine im
}
@Override
- public Writable call(Class<?> protocol, Writable writableRequest,
+ public Writable call(String protocol, Writable writableRequest,
long receiveTime) throws IOException {
ProtoSpecificRequestWritable request = (ProtoSpecificRequestWritable) writableRequest;
ProtoSpecificRpcRequest rpcRequest = request.message;
String methodName = rpcRequest.getMethodName();
- System.out.println("Call: protocol=" + protocol.getCanonicalName() + ", method="
+ System.out.println("Call: protocol=" + protocol + ", method="
+ methodName);
if (verbose)
- log("Call: protocol=" + protocol.getCanonicalName() + ", method="
+ log("Call: protocol=" + protocol + ", method="
+ methodName);
MethodDescriptor methodDescriptor = service.getDescriptorForType()
.findMethodByName(methodName);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Thu Sep 8 01:39:07 2011
@@ -50,7 +50,7 @@ public class ConverterUtils {
*
* @param url
* url to convert
- * @return
+ * @return path from {@link URL}
* @throws URISyntaxException
*/
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
@@ -63,8 +63,8 @@ public class ConverterUtils {
/**
* change from CharSequence to string for map key and value
- * @param env
- * @return
+ * @param env map for converting
+ * @return string,string map
*/
public static Map<String, String> convertToString(
Map<CharSequence, CharSequence> env) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Thu Sep 8 01:39:07 2011
@@ -221,8 +221,7 @@ public class ProcfsBasedProcessTree {
}
/** Verify that the given process id is same as its process group id.
- * @param pidStr Process id of the to-be-verified-process
- * @param procfsDir Procfs root dir
+ * @return true if the process id matches else return false.
*/
public boolean checkPidPgrpidForMatch() {
return checkPidPgrpidForMatch(pid, PROCFS);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java Thu Sep 8 01:39:07 2011
@@ -62,7 +62,7 @@ public class RackResolver {
* right resolver implementation.
* @param conf
* @param hostName
- * @return
+ * @return node {@link Node} after resolving the hostname
*/
public static Node resolve(Configuration conf, String hostName) {
init(conf);
@@ -74,7 +74,7 @@ public class RackResolver {
* network topology. This method doesn't initialize the class.
* Call {@link #init(Configuration)} explicitly.
* @param hostName
- * @return
+ * @return node {@link Node} after resolving the hostname
*/
public static Node resolve(String hostName) {
if (!initCalled) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Thu Sep 8 01:39:07 2011
@@ -77,11 +77,18 @@ public abstract class ContainerExecutor
List<Path> localDirs)
throws IOException, InterruptedException;
+
/**
* Launch the container on the node. This is a blocking call and returns only
* when the container exits.
- *
- * @param launchCtxt
+ * @param container the container to be launched
+ * @param nmPrivateContainerScriptPath the path for launch script
+ * @param nmPrivateTokensPath the path for tokens for the container
+ * @param user the user of the container
+ * @param appId the appId of the container
+ * @param containerWorkDir the work dir for the container
+ * @return the return status of the launch
+ * @throws IOException
*/
public abstract int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java Thu Sep 8 01:39:07 2011
@@ -60,11 +60,14 @@ public class DeletionService extends Abs
this.exec = exec;
this.debugDelay = 0;
}
-
+
+ /**
+ *
/**
* Delete the path(s) as this user.
* @param user The user to delete as, or the JVM user if null
- * @param p Paths to delete
+ * @param subDir the sub directory name
+ * @param baseDirs the base directories which contains the subDir's
*/
public void delete(String user, Path subDir, Path... baseDirs) {
// TODO if parent owned by NM, rename within parent inline
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Thu Sep 8 01:39:07 2011
@@ -26,11 +26,14 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.NodeHealthCheckerService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -49,6 +52,7 @@ import org.apache.hadoop.yarn.service.Co
import org.apache.hadoop.yarn.service.Service;
public class NodeManager extends CompositeService {
+ private static final Log LOG = LogFactory.getLog(NodeManager.class);
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
public NodeManager() {
@@ -185,6 +189,7 @@ public class NodeManager extends Composi
}
public static void main(String[] args) {
+ StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
NodeManager nodeManager = new NodeManager();
YarnConfiguration conf = new YarnConfiguration();
nodeManager.init(conf);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Thu Sep 8 01:39:07 2011
@@ -44,11 +44,14 @@ public class AuxServices extends Abstrac
public static final String AUX_SERVICE_CLASS_FMT =
"nodemanager.aux.service.%s.class";
public final Map<String,AuxiliaryService> serviceMap;
+ public final Map<String,ByteBuffer> serviceMeta;
public AuxServices() {
super(AuxServices.class.getName());
serviceMap =
Collections.synchronizedMap(new HashMap<String,AuxiliaryService>());
+ serviceMeta =
+ Collections.synchronizedMap(new HashMap<String,ByteBuffer>());
// Obtain services from configuration in init()
}
@@ -63,6 +66,15 @@ public class AuxServices extends Abstrac
return Collections.unmodifiableCollection(serviceMap.values());
}
+ /**
+ * @return the meta data for all registered services, that have been started.
+ * If a service has not been started no metadata will be available. The key
+ * the the name of the service as defined in the configuration.
+ */
+ public Map<String, ByteBuffer> getMeta() {
+ return Collections.unmodifiableMap(serviceMeta);
+ }
+
@Override
public void init(Configuration conf) {
Collection<String> auxNames = conf.getStringCollection(AUX_SERVICES);
@@ -75,7 +87,15 @@ public class AuxServices extends Abstrac
throw new RuntimeException("No class defiend for " + sName);
}
AuxiliaryService s = ReflectionUtils.newInstance(sClass, conf);
- // TODO better use use s.getName()?
+ // TODO better use s.getName()?
+ if(!sName.equals(s.getName())) {
+ LOG.warn("The Auxilurary Service named '"+sName+"' in the "
+ +"configuration is for class "+sClass+" which has "
+ +"a name of '"+s.getName()+"'. Because these are "
+ +"not the same tools trying to send ServiceData and read "
+ +"Service Meta Data may have issues unless the refer to "
+ +"the name in the config.");
+ }
addService(sName, s);
s.init(conf);
} catch (RuntimeException e) {
@@ -90,9 +110,15 @@ public class AuxServices extends Abstrac
public void start() {
// TODO fork(?) services running as configured user
// monitor for health, shutdown/restart(?) if any should die
- for (Service service : serviceMap.values()) {
+ for (Map.Entry<String, AuxiliaryService> entry : serviceMap.entrySet()) {
+ AuxiliaryService service = entry.getValue();
+ String name = entry.getKey();
service.start();
service.register(this);
+ ByteBuffer meta = service.getMeta();
+ if(meta != null) {
+ serviceMeta.put(name, meta);
+ }
}
super.start();
}
@@ -108,6 +134,7 @@ public class AuxServices extends Abstrac
}
}
serviceMap.clear();
+ serviceMeta.clear();
}
} finally {
super.stop();
@@ -146,6 +173,15 @@ public class AuxServices extends Abstrac
public interface AuxiliaryService extends Service {
void initApp(String user, ApplicationId appId, ByteBuffer data);
void stopApp(ApplicationId appId);
+ /**
+ * Retreive metadata for this service. This is likely going to be contact
+ * information so that applications can access the service remotely. Ideally
+ * each service should provide a method to parse out the information to a usable
+ * class. This will only be called after the services start method has finished.
+ * the result may be cached.
+ * @return metadata for this service that should be made avaiable to applications.
+ */
+ ByteBuffer getMeta();
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Thu Sep 8 01:39:07 2011
@@ -65,6 +65,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
@@ -266,6 +268,10 @@ public class ContainerManagerImpl extend
ContainerId containerID = launchContext.getContainerId();
ApplicationId applicationID = containerID.getAppId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
+ NMAuditLogger.logFailure(launchContext.getUser(),
+ AuditConstants.START_CONTAINER, "ContainerManagerImpl",
+ "Container already running on this node!",
+ applicationID, containerID);
throw RPCUtil.getRemoteException("Container " + containerID
+ " already is running on this node!!");
}
@@ -281,8 +287,14 @@ public class ContainerManagerImpl extend
// TODO: Validate the request
dispatcher.getEventHandler().handle(new ApplicationInitEvent(container));
+
+ NMAuditLogger.logSuccess(launchContext.getUser(),
+ AuditConstants.START_CONTAINER, "ContainerManageImpl",
+ applicationID, containerID);
+
StartContainerResponse response =
recordFactory.newRecordInstance(StartContainerResponse.class);
+ response.addAllServiceResponse(auxiluaryServices.getMeta());
metrics.launchedContainer();
metrics.allocateContainer(launchContext.getResource());
return response;
@@ -299,12 +311,23 @@ public class ContainerManagerImpl extend
Container container = this.context.getContainers().get(containerID);
if (container == null) {
LOG.warn("Trying to stop unknown container " + containerID);
+ NMAuditLogger.logFailure(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManagerImpl",
+ "Trying to stop unknown container!",
+ containerID.getAppId(), containerID);
return response; // Return immediately.
}
dispatcher.getEventHandler().handle(
new ContainerKillEvent(containerID,
"Container killed by the ApplicationMaster."));
+ // user logged here not ideal since just getting user from container but
+ // request doesn't have anything and should be coming from user of AM so
+ // should be the same or should be rejected by auth before here.
+ NMAuditLogger.logSuccess(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManageImpl",
+ containerID.getAppId(), containerID);
+
// TODO: Move this code to appropriate place once kill_container is
// implemented.
nodeStatusUpdater.sendOutofBandHeartBeat();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Thu Sep 8 01:39:07 2011
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
@@ -365,18 +367,28 @@ public class ContainerImpl implements Co
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
metrics.completedContainer();
+ NMAuditLogger.logSuccess(getUser(),
+ AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
+ getContainerID().getAppId(), getContainerID());
break;
case EXITED_WITH_FAILURE:
metrics.endRunningContainer();
// fall through
case LOCALIZATION_FAILED:
metrics.failedContainer();
+ NMAuditLogger.logFailure(getUser(),
+ AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
+ "Container failed with state: " + getContainerState(),
+ getContainerID().getAppId(), getContainerID());
break;
case CONTAINER_CLEANEDUP_AFTER_KILL:
metrics.endRunningContainer();
// fall through
case NEW:
metrics.killedContainer();
+ NMAuditLogger.logSuccess(getUser(),
+ AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
+ getContainerID().getAppId(), getContainerID());
}
metrics.releaseContainer(getLaunchContext().getResource());
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Thu Sep 8 01:39:07 2011
@@ -100,7 +100,8 @@ public class ContainerLaunch implements
String appIdStr = app.toString();
Path containerLogDir =
this.logDirsSelector.getLocalPathForWrite(appIdStr + Path.SEPARATOR
- + containerIdStr, LocalDirAllocator.SIZE_UNKNOWN, this.conf);
+ + containerIdStr, LocalDirAllocator.SIZE_UNKNOWN, this.conf,
+ false);
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
@@ -147,7 +148,7 @@ public class ContainerLaunch implements
+ Path.SEPARATOR + user + Path.SEPARATOR
+ ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+ Path.SEPARATOR + containerIdStr,
- LocalDirAllocator.SIZE_UNKNOWN, this.conf);
+ LocalDirAllocator.SIZE_UNKNOWN, this.conf, false);
try {
// /////////// Write out the container-script in the nmPrivate space.
String[] localDirs =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Thu Sep 8 01:39:07 2011
@@ -22,6 +22,7 @@ import org.junit.Test;
import static org.junit.Assert.*;
import java.nio.ByteBuffer;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -44,10 +45,16 @@ public class TestAuxServices {
private final int expected_appId;
private int remaining_init;
private int remaining_stop;
+ private ByteBuffer meta = null;
+
LightService(String name, char idef, int expected_appId) {
+ this(name, idef, expected_appId, null);
+ }
+ LightService(String name, char idef, int expected_appId, ByteBuffer meta) {
super(name);
this.idef = idef;
this.expected_appId = expected_appId;
+ this.meta = meta;
}
@Override
public void init(Configuration conf) {
@@ -71,14 +78,18 @@ public class TestAuxServices {
public void stopApp(ApplicationId appId) {
assertEquals(expected_appId, appId.getId());
}
+ @Override
+ public ByteBuffer getMeta() {
+ return meta;
+ }
}
static class ServiceA extends LightService {
- public ServiceA() { super("A", 'A', 65); }
+ public ServiceA() { super("A", 'A', 65, ByteBuffer.wrap("A".getBytes())); }
}
static class ServiceB extends LightService {
- public ServiceB() { super("B", 'B', 66); }
+ public ServiceB() { super("B", 'B', 66, ByteBuffer.wrap("B".getBytes())); }
}
@Test
@@ -139,6 +150,44 @@ public class TestAuxServices {
}
}
+
+ @Test
+ public void testAuxServicesMeta() {
+ Configuration conf = new Configuration();
+ conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+ conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+ ServiceA.class, Service.class);
+ conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+ ServiceB.class, Service.class);
+ final AuxServices aux = new AuxServices();
+ aux.init(conf);
+
+ int latch = 1;
+ for (Service s : aux.getServices()) {
+ assertEquals(INITED, s.getServiceState());
+ if (s instanceof ServiceA) { latch *= 2; }
+ else if (s instanceof ServiceB) { latch *= 3; }
+ else fail("Unexpected service type " + s.getClass());
+ }
+ assertEquals("Invalid mix of services", 6, latch);
+ aux.start();
+ for (Service s : aux.getServices()) {
+ assertEquals(STARTED, s.getServiceState());
+ }
+
+ Map<String, ByteBuffer> meta = aux.getMeta();
+ assertEquals(2, meta.size());
+ assertEquals("A", new String(meta.get("Asrv").array()));
+ assertEquals("B", new String(meta.get("Bsrv").array()));
+
+ aux.stop();
+ for (Service s : aux.getServices()) {
+ assertEquals(STOPPED, s.getServiceState());
+ }
+ }
+
+
+
@Test
public void testAuxUnexpectedStop() {
Configuration conf = new Configuration();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Thu Sep 8 01:39:07 2011
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.service.AbstractService;
public class AdminService extends AbstractService implements RMAdminProtocol {
@@ -113,40 +114,54 @@ public class AdminService extends Abstra
super.stop();
}
- private void checkAcls(String method) throws YarnRemoteException {
+ private UserGroupInformation checkAcls(String method) throws YarnRemoteException {
+ UserGroupInformation user;
try {
- UserGroupInformation user = UserGroupInformation.getCurrentUser();
- if (!adminAcl.isUserAllowed(user)) {
- LOG.warn("User " + user.getShortUserName() + " doesn't have permission" +
- " to call '" + method + "'");
-
- throw RPCUtil.getRemoteException(
- new AccessControlException("User " + user.getShortUserName() +
- " doesn't have permission" +
- " to call '" + method + "'")
- );
- }
-
- LOG.info("RM Admin: " + method + " invoked by user " +
- user.getShortUserName());
-
+ user = UserGroupInformation.getCurrentUser();
} catch (IOException ioe) {
LOG.warn("Couldn't get current user", ioe);
+
+ RMAuditLogger.logFailure("UNKNOWN", method,
+ adminAcl.toString(), "AdminService",
+ "Couldn't get current user");
throw RPCUtil.getRemoteException(ioe);
}
+
+ if (!adminAcl.isUserAllowed(user)) {
+ LOG.warn("User " + user.getShortUserName() + " doesn't have permission" +
+ " to call '" + method + "'");
+
+ RMAuditLogger.logFailure(user.getShortUserName(), method,
+ adminAcl.toString(), "AdminService",
+ AuditConstants.UNAUTHORIZED_USER);
+
+ throw RPCUtil.getRemoteException(
+ new AccessControlException("User " + user.getShortUserName() +
+ " doesn't have permission" +
+ " to call '" + method + "'")
+ );
+ }
+ LOG.info("RM Admin: " + method + " invoked by user " +
+ user.getShortUserName());
+
+ return user;
}
@Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
throws YarnRemoteException {
- checkAcls("refreshQueues");
-
+ UserGroupInformation user = checkAcls("refreshQueues");
try {
scheduler.reinitialize(conf, null, null); // ContainerTokenSecretManager can't
// be 'refreshed'
+ RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues",
+ "AdminService");
return recordFactory.newRecordInstance(RefreshQueuesResponse.class);
} catch (IOException ioe) {
LOG.info("Exception refreshing queues ", ioe);
+ RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
+ adminAcl.toString(), "AdminService",
+ "Exception refreshing queues");
throw RPCUtil.getRemoteException(ioe);
}
}
@@ -154,12 +169,17 @@ public class AdminService extends Abstra
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
throws YarnRemoteException {
- checkAcls("refreshNodes");
+ UserGroupInformation user = checkAcls("refreshNodes");
try {
this.nodesListManager.refreshNodes();
+ RMAuditLogger.logSuccess(user.getShortUserName(), "refreshNodes",
+ "AdminService");
return recordFactory.newRecordInstance(RefreshNodesResponse.class);
} catch (IOException ioe) {
LOG.info("Exception refreshing nodes ", ioe);
+ RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes",
+ adminAcl.toString(), "AdminService",
+ "Exception refreshing nodes");
throw RPCUtil.getRemoteException(ioe);
}
}
@@ -168,9 +188,11 @@ public class AdminService extends Abstra
public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
throws YarnRemoteException {
- checkAcls("refreshSuperUserGroupsConfiguration");
+ UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration");
ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration());
+ RMAuditLogger.logSuccess(user.getShortUserName(),
+ "refreshSuperUserGroupsConfiguration", "AdminService");
return recordFactory.newRecordInstance(
RefreshSuperUserGroupsConfigurationResponse.class);
@@ -179,9 +201,11 @@ public class AdminService extends Abstra
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
RefreshUserToGroupsMappingsRequest request) throws YarnRemoteException {
- checkAcls("refreshUserToGroupsMappings");
+ UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
Groups.getUserToGroupsMappingService().refresh();
+ RMAuditLogger.logSuccess(user.getShortUserName(),
+ "refreshUserToGroupsMappings", "AdminService");
return recordFactory.newRecordInstance(
RefreshUserToGroupsMappingsResponse.class);
@@ -190,12 +214,14 @@ public class AdminService extends Abstra
@Override
public RefreshAdminAclsResponse refreshAdminAcls(
RefreshAdminAclsRequest request) throws YarnRemoteException {
- checkAcls("refreshAdminAcls");
+ UserGroupInformation user = checkAcls("refreshAdminAcls");
Configuration conf = new Configuration();
adminAcl =
new AccessControlList(
conf.get(RMConfig.RM_ADMIN_ACL, RMConfig.DEFAULT_RM_ADMIN_ACL));
+ RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
+ "AdminService");
return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1166495&r1=1166494&r2=1166495&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Thu Sep 8 01:39:07 2011
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.ipc.RPCUti
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -115,11 +117,16 @@ public class ApplicationMasterService ex
ApplicationAttemptId applicationAttemptId = request
.getApplicationAttemptId();
+ ApplicationId appID = applicationAttemptId.getApplicationId();
AMResponse lastResponse = responseMap.get(applicationAttemptId);
if (lastResponse == null) {
String message = "Application doesn't exist in cache "
+ applicationAttemptId;
LOG.error(message);
+ RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(),
+ AuditConstants.REGISTER_AM, message, "ApplicationMasterService",
+ "Error in registering application master", appID,
+ applicationAttemptId);
throw RPCUtil.getRemoteException(message);
}
@@ -133,6 +140,10 @@ public class ApplicationMasterService ex
new RMAppAttemptRegistrationEvent(applicationAttemptId, request
.getHost(), request.getRpcPort(), request.getTrackingUrl()));
+ RMAuditLogger.logSuccess(this.rmContext.getRMApps().get(appID).getUser(),
+ AuditConstants.REGISTER_AM, "ApplicationMasterService", appID,
+ applicationAttemptId);
+
// Pick up min/max resource from scheduler...
RegisterApplicationMasterResponse response = recordFactory
.newRecordInstance(RegisterApplicationMasterResponse.class);