You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2014/04/28 21:40:26 UTC
svn commit: r1590766 - in
/hadoop/common/branches/HDFS-2006/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/
hadoop-yarn/hadoop-yarn-common/src/main/resources/
hadoop-yarn/hadoop-yarn-server/hadoop-yar...
Author: wang
Date: Mon Apr 28 19:40:06 2014
New Revision: 1590766
URL: http://svn.apache.org/r1590766
Log:
Merge changes from trunk
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt Mon Apr 28 19:40:06 2014
@@ -56,6 +56,11 @@ Release 2.5.0 - UNRELEASED
YARN-1870. FileInputStream is not closed in ProcfsBasedProcessTree#constructProcessSMAPInfo.
(Fengdong Yu via junping_du)
+ YARN-1970. Prepare YARN codebase for JUnit 4.11. (cnauroth)
+
+ YARN-483. Improve documentation on log aggregation in yarn-default.xml
+ (Akira Ajisaka via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -153,6 +158,9 @@ Release 2.4.1 - UNRELEASED
YARN-1932. Javascript injection on the job status page (Mit Desai via
jlowe)
+ YARN-1975. Used resources shows escaped html in CapacityScheduler and
+ FairScheduler page (Mit Desai via jlowe)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java Mon Apr 28 19:40:06 2014
@@ -28,7 +28,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ClientBaseWithFixes;
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java Mon Apr 28 19:40:06 2014
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.client;
import java.io.IOException;
import java.util.ArrayList;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java Mon Apr 28 19:40:06 2014
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.client;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.api.records.NodeId;
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon Apr 28 19:40:06 2014
@@ -733,7 +733,14 @@
</property>
<property>
- <description>Whether to enable log aggregation</description>
+ <description>Whether to enable log aggregation. Log aggregation collects
+ each container's logs and moves these logs onto a file-system, for e.g.
+ HDFS, after the application completes. Users can configure the
+ "yarn.nodemanager.remote-app-log-dir" and
+ "yarn.nodemanager.remote-app-log-dir-suffix" properties to determine
+ where these logs are moved to. Users can access the logs via the
+ Application Timeline Server.
+ </description>
<name>yarn.log-aggregation-enable</name>
<value>false</value>
</property>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java Mon Apr 28 19:40:06 2014
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.ActiveStandbyElector;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.ServiceFailedException;
@@ -85,8 +86,11 @@ public class EmbeddedElectorService exte
List<ACL> zkAcls = RMZKUtils.getZKAcls(conf);
List<ZKUtil.ZKAuthInfo> zkAuths = RMZKUtils.getZKAuths(conf);
+ int maxRetryNum = conf.getInt(
+ CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_KEY,
+ CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_DEFAULT);
elector = new ActiveStandbyElector(zkQuorum, (int) zkSessionTimeout,
- electionZNode, zkAcls, zkAuths, this);
+ electionZNode, zkAcls, zkAuths, this, maxRetryNum);
elector.ensureParentZNode();
if (!isParentZnodeSafe(clusterId)) {
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java Mon Apr 28 19:40:06 2014
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.uti
import java.util.ArrayList;
import org.apache.hadoop.util.StringUtils;
-import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -109,7 +108,7 @@ class CapacitySchedulerPage extends RmVi
_("Absolute Used Capacity:", percent(lqinfo.getAbsoluteUsedCapacity() / 100)).
_("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)).
_("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)).
- _("Used Resources:", StringEscapeUtils.escapeHtml(lqinfo.getResourcesUsed().toString())).
+ _("Used Resources:", lqinfo.getResourcesUsed().toString()).
_("Num Schedulable Applications:", Integer.toString(lqinfo.getNumActiveApplications())).
_("Num Non-Schedulable Applications:", Integer.toString(lqinfo.getNumPendingApplications())).
_("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java Mon Apr 28 19:40:06 2014
@@ -22,7 +22,6 @@ import static org.apache.hadoop.yarn.uti
import java.util.Collection;
-import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
@@ -65,20 +64,16 @@ public class FairSchedulerPage extends R
@Override
protected void render(Block html) {
ResponseInfo ri = info("\'" + qinfo.getQueueName() + "\' Queue Status").
- _("Used Resources:", StringEscapeUtils.escapeHtml(
- qinfo.getUsedResources().toString())).
+ _("Used Resources:", qinfo.getUsedResources().toString()).
_("Num Active Applications:", qinfo.getNumActiveApplications()).
_("Num Pending Applications:", qinfo.getNumPendingApplications()).
- _("Min Resources:", StringEscapeUtils.escapeHtml(
- qinfo.getMinResources().toString())).
- _("Max Resources:", StringEscapeUtils.escapeHtml(
- qinfo.getMaxResources().toString()));
+ _("Min Resources:", qinfo.getMinResources().toString()).
+ _("Max Resources:", qinfo.getMaxResources().toString());
int maxApps = qinfo.getMaxApplications();
if (maxApps < Integer.MAX_VALUE) {
ri._("Max Running Applications:", qinfo.getMaxApplications());
}
- ri._("Fair Share:", StringEscapeUtils.escapeHtml(
- qinfo.getFairShare().toString()));
+ ri._("Fair Share:", qinfo.getFairShare().toString());
html._(InfoBlock.class);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1590766&r1=1590765&r2=1590766&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Mon Apr 28 19:40:06 2014
@@ -28,7 +28,7 @@ import java.net.InetSocketAddress;
import javax.ws.rs.core.MediaType;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,6 +37,8 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
@@ -92,6 +94,12 @@ public class TestRMHA {
// Enable webapp to test web-services also
configuration.setBoolean(MockRM.ENABLE_WEBAPP, true);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ ClusterMetrics.destroy();
+ QueueMetrics.clearQueueMetrics();
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ if (ms.getSource("ClusterMetrics") != null) {
+ DefaultMetricsSystem.shutdown();
+ }
}
private void checkMonitorHealth() throws IOException {