You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/02/08 20:05:19 UTC
svn commit: r1566100 [2/3] - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop...
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Sat Feb 8 19:05:12 2014
@@ -675,13 +675,14 @@ public class TestYarnCLI {
int result = spyCli.run(new String[] { "-help" });
Assert.assertTrue(result == 0);
verify(spyCli).printUsage(any(Options.class));
+ System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl
Assert.assertEquals(createApplicationCLIHelpMessage(),
sysOutStream.toString());
sysOutStream.reset();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
result =
- cli.run(new String[] { "-kill", applicationId.toString(), "args" });
+ cli.run(new String[] {"application", "-kill", applicationId.toString(), "args" });
verify(spyCli).printUsage(any(Options.class));
Assert.assertEquals(createApplicationCLIHelpMessage(),
sysOutStream.toString());
@@ -717,7 +718,7 @@ public class TestYarnCLI {
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2);
- int result = cli.run(new String[] { "-kill", applicationId.toString() });
+ int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
assertEquals(0, result);
verify(client, times(0)).killApplication(any(ApplicationId.class));
verify(sysOut).println(
@@ -730,7 +731,7 @@ public class TestYarnCLI {
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
- result = cli.run(new String[] { "-kill", applicationId.toString() });
+ result = cli.run(new String[] { "application","-kill", applicationId.toString() });
assertEquals(0, result);
verify(client).killApplication(any(ApplicationId.class));
verify(sysOut).println("Killing application application_1234_0005");
@@ -740,7 +741,57 @@ public class TestYarnCLI {
.getApplicationReport(applicationId);
cli = createAndGetAppCLI();
try {
- cli.run(new String[] { "-kill", applicationId.toString() });
+ cli.run(new String[] { "application","-kill", applicationId.toString() });
+ Assert.fail();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof ApplicationNotFoundException);
+ Assert.assertEquals("Application with id '" + applicationId +
+ "' doesn't exist in RM.", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testMoveApplicationAcrossQueues() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+
+ ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+ newApplicationReport2);
+ int result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
+ assertEquals(0, result);
+ verify(client, times(0)).moveApplicationAcrossQueues(
+ any(ApplicationId.class), any(String.class));
+ verify(sysOut).println(
+ "Application " + applicationId + " has already finished ");
+
+ ApplicationReport newApplicationReport = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+ newApplicationReport);
+ result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
+ assertEquals(0, result);
+ verify(client).moveApplicationAcrossQueues(any(ApplicationId.class),
+ any(String.class));
+ verify(sysOut).println("Moving application application_1234_0005 to queue targetqueue");
+ verify(sysOut).println("Successfully completed move.");
+
+ doThrow(new ApplicationNotFoundException("Application with id '"
+ + applicationId + "' doesn't exist in RM.")).when(client)
+ .moveApplicationAcrossQueues(applicationId, "targetqueue");
+ cli = createAndGetAppCLI();
+ try {
+ result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
Assert.fail();
} catch (Exception ex) {
Assert.assertTrue(ex instanceof ApplicationNotFoundException);
@@ -1087,23 +1138,28 @@ public class TestYarnCLI {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: application");
- pw.println(" -appStates <States> Works with -list to filter applications based");
- pw.println(" on input comma-separated list of application");
- pw.println(" states. The valid application state can be one");
- pw.println(" of the following:");
- pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,");
- pw.println(" FINISHED,FAILED,KILLED");
- pw.println(" -appTypes <Types> Works with -list to filter applications based");
- pw.println(" on input comma-separated list of application");
- pw.println(" types.");
- pw.println(" -help Displays help for all commands.");
- pw.println(" -kill <Application ID> Kills the application.");
- pw.println(" -list List applications from the RM. Supports");
- pw.println(" optional use of -appTypes to filter");
- pw.println(" applications based on application type, and");
- pw.println(" -appStates to filter applications based on");
- pw.println(" application state");
- pw.println(" -status <Application ID> Prints the status of the application.");
+ pw.println(" -appStates <States> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application states. The valid application");
+ pw.println(" state can be one of the following:");
+ pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
+ pw.println(" NING,FINISHED,FAILED,KILLED");
+ pw.println(" -appTypes <Types> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application types.");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -kill <Application ID> Kills the application.");
+ pw.println(" -list List applications from the RM. Supports");
+ pw.println(" optional use of -appTypes to filter");
+ pw.println(" applications based on application type,");
+ pw.println(" and -appStates to filter applications");
+ pw.println(" based on application state");
+ pw.println(" -movetoqueue <Application ID> Moves the application to a different");
+ pw.println(" queue.");
+ pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
+ pw.println(" specify which queue to move an");
+ pw.println(" application to.");
+ pw.println(" -status <Application ID> Prints the status of the application.");
pw.close();
String appsHelpStr = baos.toString("UTF-8");
return appsHelpStr;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Sat Feb 8 19:05:12 2014
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.lang.math.LongRange;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -49,6 +50,8 @@ public class GetApplicationsRequestPBImp
Set<String> queues = null;
long limit = Long.MAX_VALUE;
LongRange start = null, finish = null;
+ private Set<String> applicationTags;
+ private ApplicationsRequestScope scope;
public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder();
@@ -112,6 +115,12 @@ public class GetApplicationsRequestPBImp
};
builder.addAllApplicationStates(iterable);
}
+ if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ builder.addAllApplicationTags(this.applicationTags);
+ }
+ if (this.scope != null) {
+ builder.setScope(ProtoUtils.convertToProtoFormat(scope));
+ }
}
private void addLocalApplicationTypesToProto() {
@@ -187,12 +196,64 @@ public class GetApplicationsRequestPBImp
this.applicationTypes = applicationTypes;
}
+ private void initApplicationTags() {
+ if (this.applicationTags != null) {
+ return;
+ }
+ GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
+ this.applicationTags = new HashSet<String>();
+ this.applicationTags.addAll(p.getApplicationTagsList());
+ }
+
+ @Override
+ public Set<String> getApplicationTags() {
+ initApplicationTags();
+ return this.applicationTags;
+ }
+
+ @Override
+ public void setApplicationTags(Set<String> tags) {
+ maybeInitBuilder();
+ if (tags == null || tags.isEmpty()) {
+ builder.clearApplicationTags();
+ this.applicationTags = null;
+ return;
+ }
+ // Convert applicationTags to lower case and add
+ this.applicationTags = new HashSet<String>();
+ for (String tag : tags) {
+ this.applicationTags.add(tag.toLowerCase());
+ }
+ }
+
@Override
public EnumSet<YarnApplicationState> getApplicationStates() {
initApplicationStates();
return this.applicationStates;
}
+ private void initScope() {
+ if (this.scope != null) {
+ return;
+ }
+ GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
+ this.scope = ProtoUtils.convertFromProtoFormat(p.getScope());
+ }
+
+ @Override
+ public ApplicationsRequestScope getScope() {
+ initScope();
+ return this.scope;
+ }
+
+ public void setScope(ApplicationsRequestScope scope) {
+ maybeInitBuilder();
+ if (scope == null) {
+ builder.clearScope();
+ }
+ this.scope = scope;
+ }
+
@Override
public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
maybeInitBuilder();
@@ -223,7 +284,6 @@ public class GetApplicationsRequestPBImp
return this.users;
}
- @Override
public void setUsers(Set<String> users) {
maybeInitBuilder();
if (users == null) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Sat Feb 8 19:05:12 2014
@@ -38,6 +38,9 @@ import org.apache.hadoop.yarn.proto.Yarn
import com.google.protobuf.TextFormat;
+import java.util.HashSet;
+import java.util.Set;
+
@Private
@Unstable
public class ApplicationReportPBImpl extends ApplicationReport {
@@ -49,6 +52,7 @@ public class ApplicationReportPBImpl ext
private ApplicationAttemptId currentApplicationAttemptId;
private Token clientToAMToken = null;
private Token amRmToken = null;
+ private Set<String> applicationTags = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@@ -245,6 +249,21 @@ public class ApplicationReportPBImpl ext
return amRmToken;
}
+ private void initApplicationTags() {
+ if (this.applicationTags != null) {
+ return;
+ }
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ this.applicationTags = new HashSet<String>();
+ this.applicationTags.addAll(p.getApplicationTagsList());
+ }
+
+ @Override
+ public Set<String> getApplicationTags() {
+ initApplicationTags();
+ return this.applicationTags;
+ }
+
@Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
@@ -356,6 +375,15 @@ public class ApplicationReportPBImpl ext
}
@Override
+ public void setApplicationTags(Set<String> tags) {
+ maybeInitBuilder();
+ if (tags == null || tags.isEmpty()) {
+ builder.clearApplicationTags();
+ }
+ this.applicationTags = tags;
+ }
+
+ @Override
public void setDiagnostics(String diagnostics) {
maybeInitBuilder();
if (diagnostics == null) {
@@ -450,6 +478,9 @@ public class ApplicationReportPBImpl ext
builder.getAmRmToken())) {
builder.setAmRmToken(convertToProtoFormat(this.amRmToken));
}
+ if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ builder.addAllApplicationTags(this.applicationTags);
+ }
}
private void mergeLocalToProto() {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Sat Feb 8 19:05:12 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
+import com.google.common.base.CharMatcher;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -25,6 +26,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
@@ -34,6 +36,9 @@ import org.apache.hadoop.yarn.proto.Yarn
import com.google.protobuf.TextFormat;
+import java.util.HashSet;
+import java.util.Set;
+
@Private
@Unstable
public class ApplicationSubmissionContextPBImpl
@@ -47,6 +52,7 @@ extends ApplicationSubmissionContext {
private Priority priority = null;
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
+ private Set<String> applicationTags = null;
public ApplicationSubmissionContextPBImpl() {
builder = ApplicationSubmissionContextProto.newBuilder();
@@ -100,6 +106,9 @@ extends ApplicationSubmissionContext {
builder.getResource())) {
builder.setResource(convertToProtoFormat(this.resource));
}
+ if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ builder.addAllApplicationTags(this.applicationTags);
+ }
}
private void mergeLocalToProto() {
@@ -196,7 +205,22 @@ extends ApplicationSubmissionContext {
}
return (p.getApplicationType());
}
-
+
+ private void initApplicationTags() {
+ if (this.applicationTags != null) {
+ return;
+ }
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ this.applicationTags = new HashSet<String>();
+ this.applicationTags.addAll(p.getApplicationTagsList());
+ }
+
+ @Override
+ public Set<String> getApplicationTags() {
+ initApplicationTags();
+ return this.applicationTags;
+ }
+
@Override
public void setQueue(String queue) {
maybeInitBuilder();
@@ -217,6 +241,40 @@ extends ApplicationSubmissionContext {
builder.setApplicationType((applicationType));
}
+ private void checkTags(Set<String> tags) {
+ if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) {
+ throw new IllegalArgumentException("Too many applicationTags, a maximum of only "
+ + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!");
+ }
+ for (String tag : tags) {
+ if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) {
+ throw new IllegalArgumentException("Tag " + tag + " is too long, " +
+ "maximum allowed length of a tag is " +
+ YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
+ }
+ if (!CharMatcher.ASCII.matchesAllOf(tag)) {
+ throw new IllegalArgumentException("A tag can only have ASCII " +
+ "characters! Invalid tag - " + tag);
+ }
+ }
+ }
+
+ @Override
+ public void setApplicationTags(Set<String> tags) {
+ maybeInitBuilder();
+ if (tags == null || tags.isEmpty()) {
+ builder.clearApplicationTags();
+ this.applicationTags = null;
+ return;
+ }
+ checkTags(tags);
+ // Convert applicationTags to lower case and add
+ this.applicationTags = new HashSet<String>();
+ for (String tag : tags) {
+ this.applicationTags.add(tag.toLowerCase());
+ }
+ }
+
@Override
public ContainerLaunchContext getAMContainerSpec() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Sat Feb 8 19:05:12 2014
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos;
@Private
@Unstable
@@ -114,6 +116,18 @@ public class ProtoUtils {
}
/*
+ * ApplicationsRequestScope
+ */
+ public static YarnServiceProtos.ApplicationsRequestScopeProto
+ convertToProtoFormat(ApplicationsRequestScope e) {
+ return YarnServiceProtos.ApplicationsRequestScopeProto.valueOf(e.name());
+ }
+ public static ApplicationsRequestScope convertFromProtoFormat
+ (YarnServiceProtos.ApplicationsRequestScopeProto e) {
+ return ApplicationsRequestScope.valueOf(e.name());
+ }
+
+ /*
* ApplicationResourceUsageReport
*/
public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat Feb 8 19:05:12 2014
@@ -130,7 +130,7 @@
<property>
<description>Are acls enabled.</description>
<name>yarn.acl.enable</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -395,7 +395,9 @@
the Active mode when prompted to.
(2) The nodes in the RM ensemble are listed in
yarn.resourcemanager.ha.rm-ids
- (3) The id of each RM comes from yarn.resourcemanager.ha.id
+ (3) The id of each RM either comes from yarn.resourcemanager.ha.id
+ if yarn.resourcemanager.ha.id is explicitly specified or can be
+ figured out by matching yarn.resourcemanager.address.{id} with local address
(4) The actual physical addresses come from the configs of the pattern
- {rpc-config}.{id}</description>
<name>yarn.resourcemanager.ha.enabled</name>
@@ -403,17 +405,20 @@
</property>
<property>
- <description>Enable automatic failover.</description>
+ <description>Enable automatic failover.
+ By default, it is enabled only when HA is enabled</description>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
- <description>Enable embedded automatic failover. The embedded elector
- relies on the RM state store to handle fencing, and is primarily intended
- to be used in conjunction with ZKRMStateStore.</description>
+ <description>Enable embedded automatic failover.
+ By default, it is enabled only when HA is enabled.
+ The embedded elector relies on the RM state store to handle fencing,
+ and is primarily intended to be used in conjunction with ZKRMStateStore.
+ </description>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -442,7 +447,10 @@
<property>
<description>The id (string) of the current RM. When HA is enabled, this
- is a required config. See description of yarn.resourcemanager.ha.enabled
+ is an optional config. The id of current RM can be set by explicitly
+ specifying yarn.resourcemanager.ha.id or figured out by matching
+ yarn.resourcemanager.address.{id} with local address
+ See description of yarn.resourcemanager.ha.enabled
for full details on how this is used.</description>
<name>yarn.resourcemanager.ha.id</name>
<!--value>rm1</value-->
@@ -588,6 +596,18 @@
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
+ <property>
+ <description>The class to use as the configuration provider.
+ If org.apache.hadoop.yarn.LocalConfigurationProvider is used,
+ the local configuration will be loaded.
+ If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used,
+ the configuration which will be loaded should be uploaded to remote File system first.
+ </description>>
+ <name>yarn.resourcemanager.configuration.provider-class</name>
+ <value>org.apache.hadoop.yarn.LocalConfigurationProvider</value>
+ <!-- <value>org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider</value> -->
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>The hostname of the NM.</description>
@@ -1120,6 +1140,20 @@
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
</property>
+ <!-- Application Timeline Service's Configuration-->
+
+ <property>
+ <description>Store class name for application timeline store</description>
+ <name>yarn.ats.store.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore</value>
+ </property>
+
+ <property>
+ <description>Store file name for leveldb application timeline store</description>
+ <name>yarn.ats.leveldb-apptimeline-store.path</name>
+ <value>${yarn.log.dir}/ats</value>
+ </property>
+
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Sat Feb 8 19:05:12 2014
@@ -159,8 +159,9 @@ public class TestHAUtil {
String confKey =
HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
assertEquals("YarnRuntimeException by Configuration#set()",
- HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(confKey),
- e.getMessage());
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(
+ HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID)
+ + " or " + confKey), e.getMessage());
}
// simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Sat Feb 8 19:05:12 2014
@@ -167,6 +167,25 @@
<artifactId>jersey-test-framework-grizzly2</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </dependency>
</dependencies>
</project>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Sat Feb 8 19:05:12 2014
@@ -27,11 +27,14 @@ import org.apache.hadoop.metrics2.source
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -51,6 +54,7 @@ public class ApplicationHistoryServer ex
ApplicationHistoryClientService ahsClientService;
ApplicationHistoryManager historyManager;
+ ApplicationTimelineStore timelineStore;
private WebApp webApp;
public ApplicationHistoryServer() {
@@ -63,6 +67,8 @@ public class ApplicationHistoryServer ex
ahsClientService = createApplicationHistoryClientService(historyManager);
addService(ahsClientService);
addService((Service) historyManager);
+ timelineStore = createApplicationTimelineStore(conf);
+ addIfService(timelineStore);
super.serviceInit(conf);
}
@@ -135,6 +141,15 @@ public class ApplicationHistoryServer ex
return new ApplicationHistoryManagerImpl();
}
+ protected ApplicationTimelineStore createApplicationTimelineStore(
+ Configuration conf) {
+ // TODO: need to replace the MemoryApplicationTimelineStore.class with the
+ // LevelDB implementation
+ return ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class,
+ ApplicationTimelineStore.class), conf);
+ }
+
protected void startWebApp() {
String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig());
LOG.info("Instantiating AHSWebApp at " + bindAddress);
@@ -148,7 +163,8 @@ public class ApplicationHistoryServer ex
YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
- .at(bindAddress).start(new AHSWebApp(historyManager));
+ .at(bindAddress)
+ .start(new AHSWebApp(historyManager, timelineStore));
} catch (Exception e) {
String msg = "AHSWebApp failed to start.";
LOG.error(msg, e);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Sat Feb 8 19:05:12 2014
@@ -21,24 +21,31 @@ import static org.apache.hadoop.yarn.uti
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
public class AHSWebApp extends WebApp implements YarnWebParams {
private final ApplicationHistoryManager applicationHistoryManager;
+ private final ApplicationTimelineStore applicationTimelineStore;
- public AHSWebApp(ApplicationHistoryManager applicationHistoryManager) {
+ public AHSWebApp(ApplicationHistoryManager applicationHistoryManager,
+ ApplicationTimelineStore applicationTimelineStore) {
this.applicationHistoryManager = applicationHistoryManager;
+ this.applicationTimelineStore = applicationTimelineStore;
}
@Override
public void setup() {
- bind(JAXBContextResolver.class);
+ bind(YarnJacksonJaxbJsonProvider.class);
bind(AHSWebServices.class);
+ bind(ATSWebServices.class);
bind(GenericExceptionHandler.class);
bind(ApplicationContext.class).toInstance(applicationHistoryManager);
+ bind(ApplicationTimelineStore.class).toInstance(applicationTimelineStore);
route("/", AHSController.class);
route(pajoin("/apps", APP_STATE), AHSController.class);
route(pajoin("/app", APPLICATION_ID), AHSController.class, "app");
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java Sat Feb 8 19:05:12 2014
@@ -40,7 +40,7 @@ public class TestApplicationHistoryServe
Configuration config = new YarnConfiguration();
historyServer.init(config);
assertEquals(STATE.INITED, historyServer.getServiceState());
- assertEquals(2, historyServer.getServices().size());
+ assertEquals(3, historyServer.getServices().size());
ApplicationHistoryClientService historyService =
historyServer.getClientService();
assertNotNull(historyServer.getClientService());
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java Sat Feb 8 19:05:12 2014
@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.Text;
@@ -312,7 +313,7 @@ public class BuilderUtils {
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
- float progress, String appType, Token amRmToken) {
+ float progress, String appType, Token amRmToken, Set<String> tags) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@@ -334,6 +335,7 @@ public class BuilderUtils {
report.setProgress(progress);
report.setApplicationType(appType);
report.setAMRMToken(amRmToken);
+ report.setApplicationTags(tags);
return report;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Sat Feb 8 19:05:12 2014
@@ -73,7 +73,8 @@ public class NonAggregatingLogHandler ex
protected void serviceInit(Configuration conf) throws Exception {
// Default 3 hours.
this.deleteDelaySeconds =
- conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 3 * 60 * 60);
+ conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
+ YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
sched = createScheduledThreadPoolExecutor(conf);
super.serviceInit(conf);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java Sat Feb 8 19:05:12 2014
@@ -145,7 +145,8 @@ public class TestNonAggregatingLogHandle
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
- conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 10800l);
+ conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
+ YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
DrainDispatcher dispatcher = createDispatcher(conf);
EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Sat Feb 8 19:05:12 2014
@@ -45,8 +45,11 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -72,6 +75,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
public class AdminService extends CompositeService implements
@@ -89,6 +93,8 @@ public class AdminService extends Compos
private InetSocketAddress masterServiceAddress;
private AccessControlList adminAcl;
+ private ConfigurationProvider configurationProvider = null;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -109,6 +115,10 @@ public class AdminService extends Compos
}
}
+ this.configurationProvider =
+ ConfigurationProviderFactory.getConfigurationProvider(conf);
+ configurationProvider.init(conf);
+
masterServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
@@ -129,6 +139,9 @@ public class AdminService extends Compos
@Override
protected synchronized void serviceStop() throws Exception {
stopServer();
+ if (this.configurationProvider != null) {
+ configurationProvider.close();
+ }
super.serviceStop();
}
@@ -295,23 +308,28 @@ public class AdminService extends Compos
@Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshQueues");
+ String argName = "refreshQueues";
+ UserGroupInformation user = checkAcls(argName);
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh queues.");
throwStandbyException();
}
+ RefreshQueuesResponse response =
+ recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- rmContext.getScheduler().reinitialize(getConfig(), this.rmContext);
- RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues",
+ Configuration conf =
+ getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE);
+ rmContext.getScheduler().reinitialize(conf, this.rmContext);
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
- return recordFactory.newRecordInstance(RefreshQueuesResponse.class);
+ return response;
} catch (IOException ioe) {
LOG.info("Exception refreshing queues ", ioe);
- RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"Exception refreshing queues");
throw RPCUtil.getRemoteException(ioe);
@@ -346,21 +364,22 @@ public class AdminService extends Compos
@Override
public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
- throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration");
+ throws YarnException, IOException {
+ String argName = "refreshSuperUserGroupsConfiguration";
+ UserGroupInformation user = checkAcls(argName);
- // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(),
- "refreshSuperUserGroupsConfiguration",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh super-user-groups.");
throwStandbyException();
}
- ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration());
+ Configuration conf =
+ getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
- "refreshSuperUserGroupsConfiguration", "AdminService");
+ argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshSuperUserGroupsConfigurationResponse.class);
@@ -391,14 +410,22 @@ public class AdminService extends Compos
@Override
public RefreshAdminAclsResponse refreshAdminAcls(
- RefreshAdminAclsRequest request) throws YarnException {
- UserGroupInformation user = checkAcls("refreshAdminAcls");
+ RefreshAdminAclsRequest request) throws YarnException, IOException {
+ String argName = "refreshAdminAcls";
+ UserGroupInformation user = checkAcls(argName);
- Configuration conf = new Configuration();
+ if (!isRMActive()) {
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
+ adminAcl.toString(), "AdminService",
+ "ResourceManager is not active. Can not refresh user-groups.");
+ throwStandbyException();
+ }
+ Configuration conf =
+ getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE);
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
- RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class);
@@ -406,9 +433,8 @@ public class AdminService extends Compos
@Override
public RefreshServiceAclsResponse refreshServiceAcls(
- RefreshServiceAclsRequest request) throws YarnException {
- Configuration conf = new Configuration();
- if (!conf.getBoolean(
+ RefreshServiceAclsRequest request) throws YarnException, IOException {
+ if (!getConfig().getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
throw RPCUtil.getRemoteException(
@@ -416,27 +442,38 @@ public class AdminService extends Compos
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION +
") not enabled."));
}
-
+
+ String argName = "refreshServiceAcls";
+ if (!isRMActive()) {
+ RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser()
+ .getShortUserName(), argName,
+ adminAcl.toString(), "AdminService",
+ "ResourceManager is not active. Can not refresh Service ACLs.");
+ throwStandbyException();
+ }
+
PolicyProvider policyProvider = new RMPolicyProvider();
-
+ Configuration conf =
+ getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+
refreshServiceAcls(conf, policyProvider);
- if (isRMActive()) {
- rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider);
- rmContext.getApplicationMasterService().refreshServiceAcls(
- conf, policyProvider);
- rmContext.getResourceTrackerService().refreshServiceAcls(
- conf, policyProvider);
- } else {
- LOG.warn("ResourceManager is not active. Not refreshing ACLs for " +
- "Clients, ApplicationMasters and NodeManagers");
- }
+ rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider);
+ rmContext.getApplicationMasterService().refreshServiceAcls(
+ conf, policyProvider);
+ rmContext.getResourceTrackerService().refreshServiceAcls(
+ conf, policyProvider);
return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class);
}
- void refreshServiceAcls(Configuration configuration,
+ synchronized void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ if (this.configurationProvider instanceof LocalConfigurationProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ } else {
+ this.server.refreshServiceAclWithConfigration(configuration,
+ policyProvider);
+ }
}
@Override
@@ -483,5 +520,19 @@ public class AdminService extends Compos
UpdateNodeResourceResponse.class);
return response;
}
-
+
+ private synchronized Configuration getConfiguration(String confFileName)
+ throws YarnException, IOException {
+ return this.configurationProvider.getConfiguration(confFileName);
+ }
+
+ @VisibleForTesting
+ public AccessControlList getAccessControlList() {
+ return this.adminAcl;
+ }
+
+ @VisibleForTesting
+ public Server getServer() {
+ return this.server;
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sat Feb 8 19:05:12 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -86,6 +87,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import com.google.common.annotations.VisibleForTesting;
+
@SuppressWarnings("unchecked")
@Private
public class ApplicationMasterService extends AbstractService implements
@@ -102,6 +105,7 @@ public class ApplicationMasterService ex
private final AllocateResponse resync =
recordFactory.newRecordInstance(AllocateResponse.class);
private final RMContext rmContext;
+ private boolean useLocalConfigurationProvider;
public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) {
super(ApplicationMasterService.class.getName());
@@ -112,6 +116,15 @@ public class ApplicationMasterService ex
}
@Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.useLocalConfigurationProvider =
+ (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
+ YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ LocalConfigurationProvider.class)));
+ super.serviceInit(conf);
+ }
+
+ @Override
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
@@ -578,7 +591,12 @@ public class ApplicationMasterService ex
public void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ if (this.useLocalConfigurationProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ } else {
+ this.server.refreshServiceAclWithConfigration(configuration,
+ policyProvider);
+ }
}
@Override
@@ -604,4 +622,9 @@ public class ApplicationMasterService ex
this.response = response;
}
}
+
+ @VisibleForTesting
+ public Server getServer() {
+ return this.server;
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Sat Feb 8 19:05:12 2014
@@ -43,7 +43,9 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
@@ -94,6 +96,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -104,6 +108,10 @@ import org.apache.hadoop.yarn.server.sec
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.SettableFuture;
+
/**
* The client interface to the Resource Manager. This module handles all the rpc
@@ -128,6 +136,7 @@ public class ClientRMService extends Abs
private final ApplicationACLsManager applicationsACLsManager;
private final QueueACLsManager queueACLsManager;
+ private boolean useLocalConfigurationProvider;
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
@@ -145,6 +154,10 @@ public class ClientRMService extends Abs
@Override
protected void serviceInit(Configuration conf) throws Exception {
clientBindAddress = getBindAddress(conf);
+ this.useLocalConfigurationProvider =
+ (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
+ YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ LocalConfigurationProvider.class)));
super.serviceInit(conf);
}
@@ -433,9 +446,11 @@ public class ClientRMService extends Abs
request.getApplicationStates();
Set<String> users = request.getUsers();
Set<String> queues = request.getQueues();
+ Set<String> tags = request.getApplicationTags();
long limit = request.getLimit();
LongRange start = request.getStartRange();
LongRange finish = request.getFinishRange();
+ ApplicationsRequestScope scope = request.getScope();
final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
Iterator<RMApp> appsIter;
@@ -482,6 +497,17 @@ public class ClientRMService extends Abs
List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
while (appsIter.hasNext() && reports.size() < limit) {
RMApp application = appsIter.next();
+
+ // Check if current application falls under the specified scope
+ boolean allowAccess = checkAccess(callerUGI, application.getUser(),
+ ApplicationAccessType.VIEW_APP, application);
+ if (scope == ApplicationsRequestScope.OWN &&
+ !callerUGI.getUserName().equals(application.getUser())) {
+ continue;
+ } else if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) {
+ continue;
+ }
+
if (applicationTypes != null && !applicationTypes.isEmpty()) {
String appTypeToMatch = caseSensitive
? application.getApplicationType()
@@ -511,8 +537,23 @@ public class ClientRMService extends Abs
continue;
}
- boolean allowAccess = checkAccess(callerUGI, application.getUser(),
- ApplicationAccessType.VIEW_APP, application);
+ if (tags != null && !tags.isEmpty()) {
+ Set<String> appTags = application.getApplicationTags();
+ if (appTags == null || appTags.isEmpty()) {
+ continue;
+ }
+ boolean match = false;
+ for (String tag : tags) {
+ if (appTags.contains(tag)) {
+ match = true;
+ break;
+ }
+ }
+ if (!match) {
+ continue;
+ }
+ }
+
reports.add(application.createAndGetApplicationReport(
callerUGI.getUserName(), allowAccess));
}
@@ -686,10 +727,74 @@ public class ClientRMService extends Abs
}
}
+ @SuppressWarnings("unchecked")
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
MoveApplicationAcrossQueuesRequest request) throws YarnException {
- throw new UnsupportedOperationException("Move not yet supported");
+ ApplicationId applicationId = request.getApplicationId();
+
+ UserGroupInformation callerUGI;
+ try {
+ callerUGI = UserGroupInformation.getCurrentUser();
+ } catch (IOException ie) {
+ LOG.info("Error getting UGI ", ie);
+ RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST,
+ "UNKNOWN", "ClientRMService" , "Error getting UGI",
+ applicationId);
+ throw RPCUtil.getRemoteException(ie);
+ }
+
+ RMApp application = this.rmContext.getRMApps().get(applicationId);
+ if (application == null) {
+ RMAuditLogger.logFailure(callerUGI.getUserName(),
+ AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
+ "Trying to move an absent application", applicationId);
+ throw new ApplicationNotFoundException("Trying to move an absent"
+ + " application " + applicationId);
+ }
+
+ if (!checkAccess(callerUGI, application.getUser(),
+ ApplicationAccessType.MODIFY_APP, application)) {
+ RMAuditLogger.logFailure(callerUGI.getShortUserName(),
+ AuditConstants.MOVE_APP_REQUEST,
+ "User doesn't have permissions to "
+ + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
+ AuditConstants.UNAUTHORIZED_USER, applicationId);
+ throw RPCUtil.getRemoteException(new AccessControlException("User "
+ + callerUGI.getShortUserName() + " cannot perform operation "
+ + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
+ }
+
+ // Moves only allowed when app is in a state that means it is tracked by
+ // the scheduler
+ if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED,
+ RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED,
+ RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED)
+ .contains(application.getState())) {
+ String msg = "App in " + application.getState() + " state cannot be moved.";
+ RMAuditLogger.logFailure(callerUGI.getShortUserName(),
+ AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
+ throw new YarnException(msg);
+ }
+
+ SettableFuture<Object> future = SettableFuture.create();
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppMoveEvent(applicationId, request.getTargetQueue(), future));
+
+ try {
+ Futures.get(future, YarnException.class);
+ } catch (YarnException ex) {
+ RMAuditLogger.logFailure(callerUGI.getShortUserName(),
+ AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
+ ex.getMessage());
+ throw ex;
+ }
+
+ RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
+ AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId);
+ MoveApplicationAcrossQueuesResponse response = recordFactory
+ .newRecordInstance(MoveApplicationAcrossQueuesResponse.class);
+ return response;
}
private String getRenewerForToken(Token<RMDelegationTokenIdentifier> token)
@@ -704,7 +809,12 @@ public class ClientRMService extends Abs
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ if (this.useLocalConfigurationProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ } else {
+ this.server.refreshServiceAclWithConfigration(configuration,
+ policyProvider);
+ }
}
private boolean isAllowedDelegationTokenOp() throws IOException {
@@ -718,4 +828,9 @@ public class ClientRMService extends Abs
return true;
}
}
+
+ @VisibleForTesting
+ public Server getServer() {
+ return this.server;
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Sat Feb 8 19:05:12 2014
@@ -320,7 +320,8 @@ public class RMAppManager implements Eve
submissionContext.getApplicationName(), user,
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
- submitTime, submissionContext.getApplicationType());
+ submitTime, submissionContext.getApplicationType(),
+ submissionContext.getApplicationTags());
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java Sat Feb 8 19:05:12 2014
@@ -45,6 +45,7 @@ public class RMAuditLogger {
public static final String KILL_APP_REQUEST = "Kill Application Request";
public static final String SUBMIT_APP_REQUEST = "Submit Application Request";
+ public static final String MOVE_APP_REQUEST = "Move Application Request";
public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
public static final String FINISH_FAILED_APP = "Application Finished - Failed";
public static final String FINISH_KILLED_APP = "Application Finished - Killed";
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sat Feb 8 19:05:12 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.VersionUtil;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -66,6 +67,8 @@ import org.apache.hadoop.yarn.server.uti
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import com.google.common.annotations.VisibleForTesting;
+
public class ResourceTrackerService extends AbstractService implements
ResourceTracker {
@@ -92,6 +95,7 @@ public class ResourceTrackerService exte
private int minAllocMb;
private int minAllocVcores;
+ private boolean useLocalConfigurationProvider;
static {
resync.setNodeAction(NodeAction.RESYNC);
@@ -141,6 +145,10 @@ public class ResourceTrackerService exte
YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);
+ this.useLocalConfigurationProvider =
+ (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
+ YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ LocalConfigurationProvider.class)));
super.serviceInit(conf);
}
@@ -415,6 +423,16 @@ public class ResourceTrackerService exte
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ if (this.useLocalConfigurationProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ } else {
+ this.server.refreshServiceAclWithConfigration(configuration,
+ policyProvider);
+ }
+ }
+
+ @VisibleForTesting
+ public Server getServer() {
+ return this.server;
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java Sat Feb 8 19:05:12 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -194,7 +195,13 @@ public interface RMApp extends EventHand
* Returns the application type
* @return the application type.
*/
- String getApplicationType();
+ String getApplicationType();
+
+ /**
+ * Get tags for the application
+ * @return tags corresponding to the application
+ */
+ Set<String> getApplicationTags();
/**
* Check whether this application is safe to terminate.
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Sat Feb 8 19:05:12 2014
@@ -23,6 +23,7 @@ public enum RMAppEventType {
START,
RECOVER,
KILL,
+ MOVE, // Move app to a new queue
// Source: Scheduler and RMAppManager
APP_REJECTED,
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Sat Feb 8 19:05:12 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
@@ -103,6 +104,7 @@ public class RMAppImpl implements RMApp,
private final long submitTime;
private final Set<RMNode> updatedNodes = new HashSet<RMNode>();
private final String applicationType;
+ private final Set<String> applicationTags;
// Mutable fields
private long startTime;
@@ -166,6 +168,8 @@ public class RMAppImpl implements RMApp,
// Transitions from SUBMITTED state
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
+ .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
+ RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING,
RMAppEventType.APP_REJECTED,
new FinalSavingTransition(
@@ -181,6 +185,8 @@ public class RMAppImpl implements RMApp,
// Transitions from ACCEPTED state
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
+ .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
+ RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING,
RMAppEventType.ATTEMPT_REGISTERED)
.addTransition(RMAppState.ACCEPTED,
@@ -190,10 +196,8 @@ public class RMAppImpl implements RMApp,
// waiting for the previous AM to exit.
RMAppEventType.ATTEMPT_FAILED,
new AttemptFailedTransition(RMAppState.ACCEPTED))
- .addTransition(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING,
- RMAppEventType.KILL,
- new FinalSavingTransition(
- new AppKilledTransition(), RMAppState.KILLED))
+ .addTransition(RMAppState.ACCEPTED, RMAppState.KILLING,
+ RMAppEventType.KILL, new KillAttemptTransition())
// ACCECPTED state can once again receive APP_ACCEPTED event, because on
// recovery the app returns ACCEPTED state and the app once again go
// through the scheduler and triggers one more APP_ACCEPTED event at
@@ -204,6 +208,8 @@ public class RMAppImpl implements RMApp,
// Transitions from RUNNING state
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
+ .addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
+ RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING,
RMAppEventType.ATTEMPT_UNREGISTERED,
new FinalSavingTransition(
@@ -295,9 +301,9 @@ public class RMAppImpl implements RMApp,
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
Configuration config, String name, String user, String queue,
- ApplicationSubmissionContext submissionContext,
- YarnScheduler scheduler,
- ApplicationMasterService masterService, long submitTime, String applicationType) {
+ ApplicationSubmissionContext submissionContext, YarnScheduler scheduler,
+ ApplicationMasterService masterService, long submitTime,
+ String applicationType, Set<String> applicationTags) {
this.applicationId = applicationId;
this.name = name;
@@ -313,6 +319,7 @@ public class RMAppImpl implements RMApp,
this.submitTime = submitTime;
this.startTime = System.currentTimeMillis();
this.applicationType = applicationType;
+ this.applicationTags = applicationTags;
int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
@@ -546,7 +553,7 @@ public class RMAppImpl implements RMApp,
createApplicationState(), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType,
- amrmToken);
+ amrmToken, applicationTags);
} finally {
this.readLock.unlock();
}
@@ -692,6 +699,31 @@ public class RMAppImpl implements RMApp,
};
}
+ /**
+ * Move an app to a new queue.
+ * This transition must set the result on the Future in the RMAppMoveEvent,
+ * either as an exception for failure or null for success, or the client will
+ * be left waiting forever.
+ */
+ @SuppressWarnings("unchecked")
+ private static final class RMAppMoveTransition extends RMAppTransition {
+ public void transition(RMAppImpl app, RMAppEvent event) {
+ RMAppMoveEvent moveEvent = (RMAppMoveEvent) event;
+ try {
+ app.queue = app.scheduler.moveApplication(app.applicationId,
+ moveEvent.getTargetQueue());
+ } catch (YarnException ex) {
+ moveEvent.getResult().setException(ex);
+ return;
+ }
+
+ // TODO: Write out change to state store (YARN-1558)
+
+ moveEvent.getResult().set(null);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
private static final class RMAppRecoveredTransition implements
MultipleArcTransition<RMAppImpl, RMAppEvent, RMAppState> {
@@ -1054,6 +1086,11 @@ public class RMAppImpl implements RMApp,
}
@Override
+ public Set<String> getApplicationTags() {
+ return this.applicationTags;
+ }
+
+ @Override
public boolean isAppSafeToTerminate() {
RMAppState state = getState();
return state.equals(RMAppState.FINISHING)
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java Sat Feb 8 19:05:12 2014
@@ -27,11 +27,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-public class AbstractYarnScheduler {
+public abstract class AbstractYarnScheduler implements ResourceScheduler {
protected RMContext rmContext;
protected Map<ApplicationId, SchedulerApplication> applications;
@@ -61,4 +62,11 @@ public class AbstractYarnScheduler {
public Map<ApplicationId, SchedulerApplication> getSchedulerApplications() {
return applications;
}
+
+ @Override
+ public String moveApplication(ApplicationId appId, String newQueue)
+ throws YarnException {
+ throw new YarnException(getClass().getSimpleName()
+ + " does not support moving apps between queues");
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java Sat Feb 8 19:05:12 2014
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.res
@Unstable
public class SchedulerApplication {
- private final Queue queue;
+ private Queue queue;
private final String user;
private SchedulerApplicationAttempt currentAttempt;
@@ -37,6 +37,10 @@ public class SchedulerApplication {
public Queue getQueue() {
return queue;
}
+
+ public void setQueue(Queue queue) {
+ this.queue = queue;
+ }
public String getUser() {
return user;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1566100&r1=1566099&r2=1566100&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Sat Feb 8 19:05:12 2014
@@ -432,7 +432,7 @@ public class SchedulerApplicationAttempt
.transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo);
}
- public void move(Queue newQueue) {
+ public synchronized void move(Queue newQueue) {
QueueMetrics oldMetrics = queue.getMetrics();
QueueMetrics newMetrics = newQueue.getMetrics();
String user = getUser();