You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/02/03 12:05:21 UTC
[1/2] hadoop git commit: YARN-4625. Make ApplicationSubmissionContext
and ApplicationSubmissionContextInfo more consistent. Contributed by Xuan
Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6941e2ccb -> e211ab777
refs/heads/trunk eb2fb943f -> 1adb64e09
YARN-4625. Make ApplicationSubmissionContext and ApplicationSubmissionContextInfo more consistent. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1adb64e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1adb64e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1adb64e0
Branch: refs/heads/trunk
Commit: 1adb64e09bd453f97e83d31b1587079e30b4b274
Parents: eb2fb94
Author: Varun Vasudev <vv...@apache.org>
Authored: Wed Feb 3 16:26:28 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Wed Feb 3 16:26:28 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 +
.../resourcemanager/webapp/RMWebServices.java | 41 ++++++-
.../webapp/dao/AMBlackListingRequestInfo.java | 61 +++++++++++
.../dao/ApplicationSubmissionContextInfo.java | 51 +++++++++
.../webapp/dao/LogAggregationContextInfo.java | 108 +++++++++++++++++++
.../TestRMWebServicesAppsModification.java | 74 ++++++++++++-
.../src/site/markdown/ResourceManagerRest.md | 54 +++++++++-
7 files changed, 386 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf93c75..87917ec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -118,6 +118,10 @@ Release 2.9.0 - UNRELEASED
YARN-4649. Add additional logging to some NM state store operations.
(Sidharta Seethana via vvasudev)
+ YARN-4625. Make ApplicationSubmissionContext and
+ ApplicationSubmissionContextInfo more consistent.
+ (Xuan Gong via vvasudev)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6c5dbd0..2ccd673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -25,7 +25,6 @@ import java.security.AccessControlException;
import java.security.Principal;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
@@ -86,6 +85,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
+import org.apache.hadoop.yarn.api.records.AMBlackListingRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -124,6 +125,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AMBlackListingRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
@@ -142,6 +144,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInf
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
@@ -1410,6 +1413,7 @@ public class RMWebServices extends WebServices {
ApplicationSubmissionContext appContext =
createAppSubmissionContext(newApp);
+
final SubmitApplicationRequest req =
SubmitApplicationRequest.newInstance(appContext);
@@ -1495,7 +1499,22 @@ public class RMWebServices extends WebServices {
newApp.getAppNodeLabelExpression(),
newApp.getAMContainerNodeLabelExpression());
appContext.setApplicationTags(newApp.getApplicationTags());
-
+ appContext.setAttemptFailuresValidityInterval(
+ newApp.getAttemptFailuresValidityInterval());
+ if (newApp.getLogAggregationContextInfo() != null) {
+ appContext.setLogAggregationContext(createLogAggregationContext(
+ newApp.getLogAggregationContextInfo()));
+ }
+ String reservationIdStr = newApp.getReservationId();
+ if (reservationIdStr != null && !reservationIdStr.isEmpty()) {
+ ReservationId reservationId = ReservationId.parseReservationId(
+ reservationIdStr);
+ appContext.setReservationID(reservationId);
+ }
+ if (newApp.getAMBlackListingRequestInfo() != null) {
+ appContext.setAMBlackListRequest(createAMBlackListingRequest(
+ newApp.getAMBlackListingRequestInfo()));
+ }
return appContext;
}
@@ -1633,6 +1652,24 @@ public class RMWebServices extends WebServices {
return callerUGI;
}
+ private LogAggregationContext createLogAggregationContext(
+ LogAggregationContextInfo logAggregationContextInfo) {
+ return LogAggregationContext.newInstance(
+ logAggregationContextInfo.getIncludePattern(),
+ logAggregationContextInfo.getExcludePattern(),
+ logAggregationContextInfo.getRolledLogsIncludePattern(),
+ logAggregationContextInfo.getRolledLogsExcludePattern(),
+ logAggregationContextInfo.getLogAggregationPolicyClassName(),
+ logAggregationContextInfo.getLogAggregationPolicyParameters());
+ }
+
+ private AMBlackListingRequest createAMBlackListingRequest(
+ AMBlackListingRequestInfo amBlackListingRequestInfo) {
+ return AMBlackListingRequest.newInstance(
+ amBlackListingRequestInfo.getAMBlackListingEnabled(),
+ amBlackListingRequestInfo.getBlackListingDisableFailureThreshold());
+ }
+
@POST
@Path("/delegation-token")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
new file mode 100644
index 0000000..0084b0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Simple class to allow users to send information required to create a
+ * AMBlackListingRequest which can then be used as part of the
+ * ApplicationSubmissionContext
+ *
+ */
+@XmlRootElement(name = "am-black-listing-requests")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AMBlackListingRequestInfo {
+
+ @XmlElement(name = "am-black-listing-enabled")
+ boolean isAMBlackListingEnabled;
+
+ @XmlElement(name = "disable-failure-threshold")
+ float disableFailureThreshold;
+
+ public AMBlackListingRequestInfo() {
+ }
+
+ public boolean getAMBlackListingEnabled() {
+ return isAMBlackListingEnabled;
+ }
+
+ public void setAMBlackListingEnabled(boolean isAMBlackListingEnabled) {
+ this.isAMBlackListingEnabled = isAMBlackListingEnabled;
+ }
+
+ public float getBlackListingDisableFailureThreshold() {
+ return disableFailureThreshold;
+ }
+
+ public void setBlackListingDisableFailureThreshold(
+ float disableFailureThreshold) {
+ this.disableFailureThreshold = disableFailureThreshold;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
index 5278b3e..4cbe7a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
@@ -78,6 +78,18 @@ public class ApplicationSubmissionContextInfo {
@XmlElement(name = "am-container-node-label-expression")
String amContainerNodeLabelExpression;
+ @XmlElement(name = "log-aggregation-context")
+ LogAggregationContextInfo logAggregationContextInfo;
+
+ @XmlElement(name = "attempt-failures-validity-interval")
+ long attemptFailuresValidityInterval;
+
+ @XmlElement(name = "reservation-id")
+ String reservationId;
+
+ @XmlElement(name = "am-black-listing-requests")
+ AMBlackListingRequestInfo amBlackListingRequestInfo;
+
public ApplicationSubmissionContextInfo() {
applicationId = "";
applicationName = "";
@@ -91,6 +103,10 @@ public class ApplicationSubmissionContextInfo {
tags = new HashSet<String>();
appNodeLabelExpression = "";
amContainerNodeLabelExpression = "";
+ logAggregationContextInfo = null;
+ attemptFailuresValidityInterval = -1;
+ reservationId = "";
+ amBlackListingRequestInfo = null;
}
public String getApplicationId() {
@@ -149,6 +165,22 @@ public class ApplicationSubmissionContextInfo {
return amContainerNodeLabelExpression;
}
+ public LogAggregationContextInfo getLogAggregationContextInfo() {
+ return logAggregationContextInfo;
+ }
+
+ public long getAttemptFailuresValidityInterval() {
+ return attemptFailuresValidityInterval;
+ }
+
+ public AMBlackListingRequestInfo getAMBlackListingRequestInfo() {
+ return amBlackListingRequestInfo;
+ }
+
+ public String getReservationId() {
+ return reservationId;
+ }
+
public void setApplicationId(String applicationId) {
this.applicationId = applicationId;
}
@@ -206,4 +238,23 @@ public class ApplicationSubmissionContextInfo {
public void setAMContainerNodeLabelExpression(String nodeLabelExpression) {
this.amContainerNodeLabelExpression = nodeLabelExpression;
}
+
+ public void setLogAggregationContextInfo(
+ LogAggregationContextInfo logAggregationContextInfo) {
+ this.logAggregationContextInfo = logAggregationContextInfo;
+ }
+
+ public void setAttemptFailuresValidityInterval(
+ long attemptFailuresValidityInterval) {
+ this.attemptFailuresValidityInterval = attemptFailuresValidityInterval;
+ }
+
+ public void setReservationId(String reservationId) {
+ this.reservationId = reservationId;
+ }
+
+ public void setAMBlackListingRequestInfo(
+ AMBlackListingRequestInfo amBlackListingRequestInfo) {
+ this.amBlackListingRequestInfo = amBlackListingRequestInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
new file mode 100644
index 0000000..d25d7b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Simple class to allow users to send information required to create a
+ * ContainerLaunchContext which can then be used as part of the
+ * ApplicationSubmissionContext
+ *
+ */
+@XmlRootElement(name = "log-aggregation-context")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LogAggregationContextInfo {
+
+ @XmlElement(name = "log-include-pattern")
+ String logIncludePattern;
+
+ @XmlElement(name = "log-exclude-pattern")
+ String logExcludePattern;
+
+ @XmlElement(name = "rolled-log-include-pattern")
+ String rolledLogsIncludePattern;
+
+ @XmlElement(name = "rolled-log-exclude-pattern")
+ String rolledLogsExcludePattern;
+
+ @XmlElement(name = "log-aggregation-policy-class-name")
+ String policyClassName;
+
+ @XmlElement(name = "log-aggregation-policy-parameters")
+ String policyParameters;
+
+ public LogAggregationContextInfo() {
+ }
+
+ public String getIncludePattern() {
+ return this.logIncludePattern;
+ }
+
+ public void setIncludePattern(String includePattern) {
+ this.logIncludePattern = includePattern;
+ }
+
+ public String getExcludePattern() {
+ return this.logExcludePattern;
+ }
+
+ public void setExcludePattern(String excludePattern) {
+ this.logExcludePattern = excludePattern;
+ }
+
+ public String getRolledLogsIncludePattern() {
+ return this.rolledLogsIncludePattern;
+ }
+
+ public void setRolledLogsIncludePattern(
+ String rolledLogsIncludePattern) {
+ this.rolledLogsIncludePattern = rolledLogsIncludePattern;
+ }
+
+ public String getRolledLogsExcludePattern() {
+ return this.rolledLogsExcludePattern;
+ }
+
+ public void setRolledLogsExcludePattern(
+ String rolledLogsExcludePattern) {
+ this.rolledLogsExcludePattern = rolledLogsExcludePattern;
+ }
+
+ public String getLogAggregationPolicyClassName() {
+ return this.policyClassName;
+ }
+
+ public void setLogAggregationPolicyClassName(
+ String className) {
+ this.policyClassName = className;
+ }
+
+ public String getLogAggregationPolicyParameters() {
+ return this.policyParameters;
+ }
+
+ public void setLogAggregationPolicyParameters(
+ String parameters) {
+ this.policyParameters = parameters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 61579f8..9abcf9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -44,18 +44,23 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.sun.jersey.api.client.config.DefaultClientConfig;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.yarn.api.records.AMBlackListingRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -67,10 +72,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -792,6 +793,47 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
appInfo.getResource().setvCores(1);
appInfo.setApplicationTags(tags);
+ // Set LogAggregationContextInfo
+ String includePattern = "file1";
+ String excludePattern = "file2";
+ String rolledLogsIncludePattern = "file3";
+ String rolledLogsExcludePattern = "file4";
+ String className = "policy_class";
+ String parameters = "policy_parameter";
+
+ LogAggregationContextInfo logAggregationContextInfo
+ = new LogAggregationContextInfo();
+ logAggregationContextInfo.setIncludePattern(includePattern);
+ logAggregationContextInfo.setExcludePattern(excludePattern);
+ logAggregationContextInfo.setRolledLogsIncludePattern(
+ rolledLogsIncludePattern);
+ logAggregationContextInfo.setRolledLogsExcludePattern(
+ rolledLogsExcludePattern);
+ logAggregationContextInfo.setLogAggregationPolicyClassName(className);
+ logAggregationContextInfo.setLogAggregationPolicyParameters(parameters);
+ appInfo.setLogAggregationContextInfo(logAggregationContextInfo);
+
+ // Set attemptFailuresValidityInterval
+ long attemptFailuresValidityInterval = 5000;
+ appInfo.setAttemptFailuresValidityInterval(
+ attemptFailuresValidityInterval);
+
+ // Set ReservationId
+ String reservationId = ReservationId.newInstance(
+ System.currentTimeMillis(), 1).toString();
+ appInfo.setReservationId(reservationId);
+
+ // Set AMBlackListingRequestInfo
+ boolean isAMBlackListingEnabled = true;
+ float disableFailureThreshold = 0.01f;
+ AMBlackListingRequestInfo amBlackListingRequestInfo
+ = new AMBlackListingRequestInfo();
+ amBlackListingRequestInfo.setAMBlackListingEnabled(
+ isAMBlackListingEnabled);
+ amBlackListingRequestInfo.setBlackListingDisableFailureThreshold(
+ disableFailureThreshold);
+ appInfo.setAMBlackListingRequestInfo(amBlackListingRequestInfo);
+
ClientResponse response =
this.constructWebResource(urlPath).accept(acceptMedia)
.entity(appInfo, contentMedia).post(ClientResponse.class);
@@ -849,6 +891,30 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
.getAllSecretKeys().contains(key));
assertEquals("mysecret", new String(cs.getSecretKey(key), "UTF-8"));
+ // Check LogAggregationContext
+ ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
+ LogAggregationContext lac = asc.getLogAggregationContext();
+ assertEquals(includePattern, lac.getIncludePattern());
+ assertEquals(excludePattern, lac.getExcludePattern());
+ assertEquals(rolledLogsIncludePattern, lac.getRolledLogsIncludePattern());
+ assertEquals(rolledLogsExcludePattern, lac.getRolledLogsExcludePattern());
+ assertEquals(className, lac.getLogAggregationPolicyClassName());
+ assertEquals(parameters, lac.getLogAggregationPolicyParameters());
+
+ // Check attemptFailuresValidityInterval
+ assertEquals(attemptFailuresValidityInterval,
+ asc.getAttemptFailuresValidityInterval());
+
+ // Check ReservationId
+ assertEquals(reservationId, app.getReservationId().toString());
+
+ // Check AMBlackListingRequestInfo
+ AMBlackListingRequest amBlackListingRequest = asc.getAMBlackListRequest();
+ assertEquals(isAMBlackListingEnabled,
+ amBlackListingRequest.isAMBlackListingEnabled());
+ assertTrue(disableFailureThreshold == amBlackListingRequest
+ .getBlackListingDisableFailureThreshold());
+
response =
this.constructWebResource("apps", appId).accept(acceptMedia)
.get(ClientResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1adb64e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 1d11ed4..aed50a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -2236,6 +2236,10 @@ Please note that this feature is currently in the alpha stage and may change in
| application-type | string | The application type(MapReduce, Pig, Hive, etc) |
| keep-containers-across-application-attempts | boolean | Should YARN keep the containers used by this application instead of destroying them |
| application-tags | object | List of application tags, please see the request examples on how to speciy the tags |
+| log-aggregation-context| object | Represents all of the information needed by the NodeManager to handle the logs for this application |
+| attempt-failures-validity-interval| long | The failure number will no take attempt failures which happen out of the validityInterval into failure count|
+| reservation-id| string | Represent the unique id of the corresponding reserved resource allocation in the scheduler |
+| am-black-listing-requests| object | Contains blacklisting information such as "enable/disable AM blacklisting" and "disable failure threshold" |
Elements of the *am-container-spec* object
@@ -2278,6 +2282,24 @@ Elements of the POST request body *resource* object
| memory | int | Memory required for each container |
| vCores | int | Virtual cores required for each container |
+Elements of the POST request body *log-aggregation-context* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| log-include-pattern | string | The log files which match the defined include pattern will be uploaded when the applicaiton finishes |
+| log-exclude-pattern | string | The log files which match the defined exclude pattern will not be uploaded when the applicaiton finishes |
+| rolled-log-include-pattern | string | The log files which match the defined include pattern will be aggregated in a rolling fashion |
+| rolled-log-exclude-pattern | string | The log files which match the defined exclude pattern will not be aggregated in a rolling fashion |
+| log-aggregation-policy-class-name | string | The policy which will be used by NodeManager to aggregate the logs |
+| log-aggregation-policy-parameters | string | The parameters passed to the policy class |
+
+Elements of the POST request body *am-black-listing-requests* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| am-black-listing-enabled | boolean | Whether AM Blacklisting is enabled |
+| disable-failure-threshold | float | AM Blacklisting disable failure threshold |
+
**JSON response**
HTTP Request:
@@ -2343,7 +2365,23 @@ HTTP Request:
"vCores":1
},
"application-type":"YARN",
- "keep-containers-across-application-attempts":false
+ "keep-containers-across-application-attempts":false,
+ "log-aggregation-context":
+ {
+ "log-include-pattern":"file1",
+ "log-exclude-pattern":"file2",
+ "rolled-log-include-pattern":"file3",
+ "rolled-log-exclude-pattern":"file4",
+ "log-aggregation-policy-class-name":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy",
+ "log-aggregation-policy-parameters":""
+ },
+ "attempt-failures-validity-interval":3600000,
+ "reservation-id":"reservation_1454114874_1",
+ "am-black-listing-requests":
+ {
+ "am-black-listing-enabled":true,
+ "disable-failure-threshold":0.01
+ }
}
```
@@ -2445,6 +2483,20 @@ Content-Type: application/xml
<tag>tag 2</tag>
<tag>tag1</tag>
</application-tags>
+ <log-aggregation-context>
+ <log-include-pattern>file1</log-include-pattern>
+ <log-exclude-pattern>file2</log-exclude-pattern>
+ <rolled-log-include-pattern>file3</rolled-log-include-pattern>
+ <rolled-log-exclude-pattern>file4</rolled-log-exclude-pattern>
+ <log-aggregation-policy-class-name>org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy</log-aggregation-policy-class-name>
+ <log-aggregation-policy-parameters></log-aggregation-policy-parameters>
+ </log-aggregation-context>
+ <attempt-failures-validity-interval>3600000</attempt-failures-validity-interval>
+ <reservation-id>reservation_1454114874_1</reservation-id>
+ <am-black-listing-requests>
+ <am-black-listing-enabled>true</am-black-listing-enabled>
+ <disable-failure-threshold>0.01</disable-failure-threshold>
+ </am-black-listing-requests>
</application-submission-context>
```
[2/2] hadoop git commit: YARN-4625. Make ApplicationSubmissionContext
and ApplicationSubmissionContextInfo more consistent. Contributed by Xuan
Gong.
Posted by vv...@apache.org.
YARN-4625. Make ApplicationSubmissionContext and ApplicationSubmissionContextInfo more consistent. Contributed by Xuan Gong.
(cherry picked from commit 1adb64e09bd453f97e83d31b1587079e30b4b274)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e211ab77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e211ab77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e211ab77
Branch: refs/heads/branch-2
Commit: e211ab777e6b84612bc8e0d26e1c4afcfd35cf1b
Parents: 6941e2c
Author: Varun Vasudev <vv...@apache.org>
Authored: Wed Feb 3 16:26:28 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Wed Feb 3 16:28:31 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 +
.../resourcemanager/webapp/RMWebServices.java | 41 ++++++-
.../webapp/dao/AMBlackListingRequestInfo.java | 61 +++++++++++
.../dao/ApplicationSubmissionContextInfo.java | 51 +++++++++
.../webapp/dao/LogAggregationContextInfo.java | 108 +++++++++++++++++++
.../TestRMWebServicesAppsModification.java | 74 ++++++++++++-
.../src/site/markdown/ResourceManagerRest.md | 54 +++++++++-
7 files changed, 386 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5968d4e..5d7b548 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -60,6 +60,10 @@ Release 2.9.0 - UNRELEASED
YARN-4649. Add additional logging to some NM state store operations.
(Sidharta Seethana via vvasudev)
+ YARN-4625. Make ApplicationSubmissionContext and
+ ApplicationSubmissionContextInfo more consistent.
+ (Xuan Gong via vvasudev)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6c5dbd0..2ccd673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -25,7 +25,6 @@ import java.security.AccessControlException;
import java.security.Principal;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
@@ -86,6 +85,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
+import org.apache.hadoop.yarn.api.records.AMBlackListingRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -124,6 +125,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AMBlackListingRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
@@ -142,6 +144,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInf
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
@@ -1410,6 +1413,7 @@ public class RMWebServices extends WebServices {
ApplicationSubmissionContext appContext =
createAppSubmissionContext(newApp);
+
final SubmitApplicationRequest req =
SubmitApplicationRequest.newInstance(appContext);
@@ -1495,7 +1499,22 @@ public class RMWebServices extends WebServices {
newApp.getAppNodeLabelExpression(),
newApp.getAMContainerNodeLabelExpression());
appContext.setApplicationTags(newApp.getApplicationTags());
-
+ appContext.setAttemptFailuresValidityInterval(
+ newApp.getAttemptFailuresValidityInterval());
+ if (newApp.getLogAggregationContextInfo() != null) {
+ appContext.setLogAggregationContext(createLogAggregationContext(
+ newApp.getLogAggregationContextInfo()));
+ }
+ String reservationIdStr = newApp.getReservationId();
+ if (reservationIdStr != null && !reservationIdStr.isEmpty()) {
+ ReservationId reservationId = ReservationId.parseReservationId(
+ reservationIdStr);
+ appContext.setReservationID(reservationId);
+ }
+ if (newApp.getAMBlackListingRequestInfo() != null) {
+ appContext.setAMBlackListRequest(createAMBlackListingRequest(
+ newApp.getAMBlackListingRequestInfo()));
+ }
return appContext;
}
@@ -1633,6 +1652,24 @@ public class RMWebServices extends WebServices {
return callerUGI;
}
+ private LogAggregationContext createLogAggregationContext(
+ LogAggregationContextInfo logAggregationContextInfo) {
+ return LogAggregationContext.newInstance(
+ logAggregationContextInfo.getIncludePattern(),
+ logAggregationContextInfo.getExcludePattern(),
+ logAggregationContextInfo.getRolledLogsIncludePattern(),
+ logAggregationContextInfo.getRolledLogsExcludePattern(),
+ logAggregationContextInfo.getLogAggregationPolicyClassName(),
+ logAggregationContextInfo.getLogAggregationPolicyParameters());
+ }
+
+ private AMBlackListingRequest createAMBlackListingRequest(
+ AMBlackListingRequestInfo amBlackListingRequestInfo) {
+ return AMBlackListingRequest.newInstance(
+ amBlackListingRequestInfo.getAMBlackListingEnabled(),
+ amBlackListingRequestInfo.getBlackListingDisableFailureThreshold());
+ }
+
@POST
@Path("/delegation-token")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
new file mode 100644
index 0000000..0084b0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Simple class to allow users to send information required to create a
+ * AMBlackListingRequest which can then be used as part of the
+ * ApplicationSubmissionContext
+ *
+ */
+@XmlRootElement(name = "am-black-listing-requests")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AMBlackListingRequestInfo {
+
+ @XmlElement(name = "am-black-listing-enabled")
+ boolean isAMBlackListingEnabled;
+
+ @XmlElement(name = "disable-failure-threshold")
+ float disableFailureThreshold;
+
+ public AMBlackListingRequestInfo() {
+ }
+
+ public boolean getAMBlackListingEnabled() {
+ return isAMBlackListingEnabled;
+ }
+
+ public void setAMBlackListingEnabled(boolean isAMBlackListingEnabled) {
+ this.isAMBlackListingEnabled = isAMBlackListingEnabled;
+ }
+
+ public float getBlackListingDisableFailureThreshold() {
+ return disableFailureThreshold;
+ }
+
+ public void setBlackListingDisableFailureThreshold(
+ float disableFailureThreshold) {
+ this.disableFailureThreshold = disableFailureThreshold;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
index 5278b3e..4cbe7a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
@@ -78,6 +78,18 @@ public class ApplicationSubmissionContextInfo {
@XmlElement(name = "am-container-node-label-expression")
String amContainerNodeLabelExpression;
+ @XmlElement(name = "log-aggregation-context")
+ LogAggregationContextInfo logAggregationContextInfo;
+
+ @XmlElement(name = "attempt-failures-validity-interval")
+ long attemptFailuresValidityInterval;
+
+ @XmlElement(name = "reservation-id")
+ String reservationId;
+
+ @XmlElement(name = "am-black-listing-requests")
+ AMBlackListingRequestInfo amBlackListingRequestInfo;
+
public ApplicationSubmissionContextInfo() {
applicationId = "";
applicationName = "";
@@ -91,6 +103,10 @@ public class ApplicationSubmissionContextInfo {
tags = new HashSet<String>();
appNodeLabelExpression = "";
amContainerNodeLabelExpression = "";
+ logAggregationContextInfo = null;
+ attemptFailuresValidityInterval = -1;
+ reservationId = "";
+ amBlackListingRequestInfo = null;
}
public String getApplicationId() {
@@ -149,6 +165,22 @@ public class ApplicationSubmissionContextInfo {
return amContainerNodeLabelExpression;
}
+ public LogAggregationContextInfo getLogAggregationContextInfo() {
+ return logAggregationContextInfo;
+ }
+
+ public long getAttemptFailuresValidityInterval() {
+ return attemptFailuresValidityInterval;
+ }
+
+ public AMBlackListingRequestInfo getAMBlackListingRequestInfo() {
+ return amBlackListingRequestInfo;
+ }
+
+ public String getReservationId() {
+ return reservationId;
+ }
+
public void setApplicationId(String applicationId) {
this.applicationId = applicationId;
}
@@ -206,4 +238,23 @@ public class ApplicationSubmissionContextInfo {
public void setAMContainerNodeLabelExpression(String nodeLabelExpression) {
this.amContainerNodeLabelExpression = nodeLabelExpression;
}
+
+ public void setLogAggregationContextInfo(
+ LogAggregationContextInfo logAggregationContextInfo) {
+ this.logAggregationContextInfo = logAggregationContextInfo;
+ }
+
+ public void setAttemptFailuresValidityInterval(
+ long attemptFailuresValidityInterval) {
+ this.attemptFailuresValidityInterval = attemptFailuresValidityInterval;
+ }
+
+ public void setReservationId(String reservationId) {
+ this.reservationId = reservationId;
+ }
+
+ public void setAMBlackListingRequestInfo(
+ AMBlackListingRequestInfo amBlackListingRequestInfo) {
+ this.amBlackListingRequestInfo = amBlackListingRequestInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
new file mode 100644
index 0000000..d25d7b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Simple class to allow users to send information required to create a
+ * ContainerLaunchContext which can then be used as part of the
+ * ApplicationSubmissionContext
+ *
+ */
+@XmlRootElement(name = "log-aggregation-context")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LogAggregationContextInfo {
+
+ @XmlElement(name = "log-include-pattern")
+ String logIncludePattern;
+
+ @XmlElement(name = "log-exclude-pattern")
+ String logExcludePattern;
+
+ @XmlElement(name = "rolled-log-include-pattern")
+ String rolledLogsIncludePattern;
+
+ @XmlElement(name = "rolled-log-exclude-pattern")
+ String rolledLogsExcludePattern;
+
+ @XmlElement(name = "log-aggregation-policy-class-name")
+ String policyClassName;
+
+ @XmlElement(name = "log-aggregation-policy-parameters")
+ String policyParameters;
+
+ public LogAggregationContextInfo() {
+ }
+
+ public String getIncludePattern() {
+ return this.logIncludePattern;
+ }
+
+ public void setIncludePattern(String includePattern) {
+ this.logIncludePattern = includePattern;
+ }
+
+ public String getExcludePattern() {
+ return this.logExcludePattern;
+ }
+
+ public void setExcludePattern(String excludePattern) {
+ this.logExcludePattern = excludePattern;
+ }
+
+ public String getRolledLogsIncludePattern() {
+ return this.rolledLogsIncludePattern;
+ }
+
+ public void setRolledLogsIncludePattern(
+ String rolledLogsIncludePattern) {
+ this.rolledLogsIncludePattern = rolledLogsIncludePattern;
+ }
+
+ public String getRolledLogsExcludePattern() {
+ return this.rolledLogsExcludePattern;
+ }
+
+ public void setRolledLogsExcludePattern(
+ String rolledLogsExcludePattern) {
+ this.rolledLogsExcludePattern = rolledLogsExcludePattern;
+ }
+
+ public String getLogAggregationPolicyClassName() {
+ return this.policyClassName;
+ }
+
+ public void setLogAggregationPolicyClassName(
+ String className) {
+ this.policyClassName = className;
+ }
+
+ public String getLogAggregationPolicyParameters() {
+ return this.policyParameters;
+ }
+
+ public void setLogAggregationPolicyParameters(
+ String parameters) {
+ this.policyParameters = parameters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 61579f8..9abcf9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -44,18 +44,23 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.sun.jersey.api.client.config.DefaultClientConfig;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.yarn.api.records.AMBlackListingRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -67,10 +72,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -792,6 +793,47 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
appInfo.getResource().setvCores(1);
appInfo.setApplicationTags(tags);
+ // Set LogAggregationContextInfo
+ String includePattern = "file1";
+ String excludePattern = "file2";
+ String rolledLogsIncludePattern = "file3";
+ String rolledLogsExcludePattern = "file4";
+ String className = "policy_class";
+ String parameters = "policy_parameter";
+
+ LogAggregationContextInfo logAggregationContextInfo
+ = new LogAggregationContextInfo();
+ logAggregationContextInfo.setIncludePattern(includePattern);
+ logAggregationContextInfo.setExcludePattern(excludePattern);
+ logAggregationContextInfo.setRolledLogsIncludePattern(
+ rolledLogsIncludePattern);
+ logAggregationContextInfo.setRolledLogsExcludePattern(
+ rolledLogsExcludePattern);
+ logAggregationContextInfo.setLogAggregationPolicyClassName(className);
+ logAggregationContextInfo.setLogAggregationPolicyParameters(parameters);
+ appInfo.setLogAggregationContextInfo(logAggregationContextInfo);
+
+ // Set attemptFailuresValidityInterval
+ long attemptFailuresValidityInterval = 5000;
+ appInfo.setAttemptFailuresValidityInterval(
+ attemptFailuresValidityInterval);
+
+ // Set ReservationId
+ String reservationId = ReservationId.newInstance(
+ System.currentTimeMillis(), 1).toString();
+ appInfo.setReservationId(reservationId);
+
+ // Set AMBlackListingRequestInfo
+ boolean isAMBlackListingEnabled = true;
+ float disableFailureThreshold = 0.01f;
+ AMBlackListingRequestInfo amBlackListingRequestInfo
+ = new AMBlackListingRequestInfo();
+ amBlackListingRequestInfo.setAMBlackListingEnabled(
+ isAMBlackListingEnabled);
+ amBlackListingRequestInfo.setBlackListingDisableFailureThreshold(
+ disableFailureThreshold);
+ appInfo.setAMBlackListingRequestInfo(amBlackListingRequestInfo);
+
ClientResponse response =
this.constructWebResource(urlPath).accept(acceptMedia)
.entity(appInfo, contentMedia).post(ClientResponse.class);
@@ -849,6 +891,30 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
.getAllSecretKeys().contains(key));
assertEquals("mysecret", new String(cs.getSecretKey(key), "UTF-8"));
+ // Check LogAggregationContext
+ ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
+ LogAggregationContext lac = asc.getLogAggregationContext();
+ assertEquals(includePattern, lac.getIncludePattern());
+ assertEquals(excludePattern, lac.getExcludePattern());
+ assertEquals(rolledLogsIncludePattern, lac.getRolledLogsIncludePattern());
+ assertEquals(rolledLogsExcludePattern, lac.getRolledLogsExcludePattern());
+ assertEquals(className, lac.getLogAggregationPolicyClassName());
+ assertEquals(parameters, lac.getLogAggregationPolicyParameters());
+
+ // Check attemptFailuresValidityInterval
+ assertEquals(attemptFailuresValidityInterval,
+ asc.getAttemptFailuresValidityInterval());
+
+ // Check ReservationId
+ assertEquals(reservationId, app.getReservationId().toString());
+
+ // Check AMBlackListingRequestInfo
+ AMBlackListingRequest amBlackListingRequest = asc.getAMBlackListRequest();
+ assertEquals(isAMBlackListingEnabled,
+ amBlackListingRequest.isAMBlackListingEnabled());
+ assertTrue(disableFailureThreshold == amBlackListingRequest
+ .getBlackListingDisableFailureThreshold());
+
response =
this.constructWebResource("apps", appId).accept(acceptMedia)
.get(ClientResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e211ab77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 1d11ed4..aed50a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -2236,6 +2236,10 @@ Please note that this feature is currently in the alpha stage and may change in
| application-type | string | The application type(MapReduce, Pig, Hive, etc) |
| keep-containers-across-application-attempts | boolean | Should YARN keep the containers used by this application instead of destroying them |
| application-tags | object | List of application tags, please see the request examples on how to speciy the tags |
+| log-aggregation-context| object | Represents all of the information needed by the NodeManager to handle the logs for this application |
+| attempt-failures-validity-interval| long | The failure number will no take attempt failures which happen out of the validityInterval into failure count|
+| reservation-id| string | Represent the unique id of the corresponding reserved resource allocation in the scheduler |
+| am-black-listing-requests| object | Contains blacklisting information such as "enable/disable AM blacklisting" and "disable failure threshold" |
Elements of the *am-container-spec* object
@@ -2278,6 +2282,24 @@ Elements of the POST request body *resource* object
| memory | int | Memory required for each container |
| vCores | int | Virtual cores required for each container |
+Elements of the POST request body *log-aggregation-context* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| log-include-pattern | string | The log files which match the defined include pattern will be uploaded when the applicaiton finishes |
+| log-exclude-pattern | string | The log files which match the defined exclude pattern will not be uploaded when the applicaiton finishes |
+| rolled-log-include-pattern | string | The log files which match the defined include pattern will be aggregated in a rolling fashion |
+| rolled-log-exclude-pattern | string | The log files which match the defined exclude pattern will not be aggregated in a rolling fashion |
+| log-aggregation-policy-class-name | string | The policy which will be used by NodeManager to aggregate the logs |
+| log-aggregation-policy-parameters | string | The parameters passed to the policy class |
+
+Elements of the POST request body *am-black-listing-requests* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| am-black-listing-enabled | boolean | Whether AM Blacklisting is enabled |
+| disable-failure-threshold | float | AM Blacklisting disable failure threshold |
+
**JSON response**
HTTP Request:
@@ -2343,7 +2365,23 @@ HTTP Request:
"vCores":1
},
"application-type":"YARN",
- "keep-containers-across-application-attempts":false
+ "keep-containers-across-application-attempts":false,
+ "log-aggregation-context":
+ {
+ "log-include-pattern":"file1",
+ "log-exclude-pattern":"file2",
+ "rolled-log-include-pattern":"file3",
+ "rolled-log-exclude-pattern":"file4",
+ "log-aggregation-policy-class-name":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy",
+ "log-aggregation-policy-parameters":""
+ },
+ "attempt-failures-validity-interval":3600000,
+ "reservation-id":"reservation_1454114874_1",
+ "am-black-listing-requests":
+ {
+ "am-black-listing-enabled":true,
+ "disable-failure-threshold":0.01
+ }
}
```
@@ -2445,6 +2483,20 @@ Content-Type: application/xml
<tag>tag 2</tag>
<tag>tag1</tag>
</application-tags>
+ <log-aggregation-context>
+ <log-include-pattern>file1</log-include-pattern>
+ <log-exclude-pattern>file2</log-exclude-pattern>
+ <rolled-log-include-pattern>file3</rolled-log-include-pattern>
+ <rolled-log-exclude-pattern>file4</rolled-log-exclude-pattern>
+ <log-aggregation-policy-class-name>org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy</log-aggregation-policy-class-name>
+ <log-aggregation-policy-parameters></log-aggregation-policy-parameters>
+ </log-aggregation-context>
+ <attempt-failures-validity-interval>3600000</attempt-failures-validity-interval>
+ <reservation-id>reservation_1454114874_1</reservation-id>
+ <am-black-listing-requests>
+ <am-black-listing-enabled>true</am-black-listing-enabled>
+ <disable-failure-threshold>0.01</disable-failure-threshold>
+ </am-black-listing-requests>
</application-submission-context>
```