You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2014/05/19 21:29:35 UTC
svn commit: r1596000 - in
/hadoop/common/branches/HDFS-2006/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api...
Author: wang
Date: Mon May 19 19:29:30 2014
New Revision: 1596000
URL: http://svn.apache.org/r1596000
Log:
Merge trunk r1595999 to branch.
Removed:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt Mon May 19 19:29:30 2014
@@ -111,12 +111,6 @@ Release 2.5.0 - UNRELEASED
YARN-2011. Fix typo and warning in TestLeafQueue (Chen He via junping_du)
- YARN-1976. Fix yarn application CLI to print the scheme of the tracking url
- of failed/killed applications. (Junping Du via jianhe)
-
- YARN-2016. Fix a bug in GetApplicationsRequestPBImpl to add the missed fields
- to proto. (Junping Du via jianhe)
-
YARN-2042. String shouldn't be compared using == in
QueuePlacementRule#NestedUserQueue#getQueueForApp (Chen He via Sandy Ryza)
@@ -228,6 +222,18 @@ Release 2.4.1 - UNRELEASED
YARN-1986. In Fifo Scheduler, node heartbeat in between creating app and
attempt causes NPE (Hong Zhiguo via Sandy Ryza)
+ YARN-1976. Fix yarn application CLI to print the scheme of the tracking url
+ of failed/killed applications. (Junping Du via jianhe)
+
+ YARN-2016. Fix a bug in GetApplicationsRequestPBImpl to add the missed fields
+ to proto. (Junping Du via jianhe)
+
+ YARN-2053. Fixed a bug in AMS to not add null NMToken into NMTokens list from
+ previous attempts for work-preserving AM restart. (Wangda Tan via jianhe)
+
+ YARN-2066. Wrong field is referenced in GetApplicationsRequestPBImpl#mergeLocalToBuilder()
+ (Hong Zhiguo via junping_du)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Mon May 19 19:29:30 2014
@@ -127,8 +127,8 @@ public class GetApplicationsRequestPBImp
}
if (this.finish != null) {
- builder.setFinishBegin(start.getMinimumLong());
- builder.setFinishEnd(start.getMaximumLong());
+ builder.setFinishBegin(finish.getMinimumLong());
+ builder.setFinishEnd(finish.getMaximumLong());
}
builder.setLimit(limit);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java Mon May 19 19:29:30 2014
@@ -47,10 +47,12 @@ public class TestGetApplicationsRequest
types.add("type1");
request.setApplicationTypes(types);
- long begin = System.currentTimeMillis();
- long end = System.currentTimeMillis() + 1;
- request.setStartRange(begin, end);
- request.setFinishRange(begin, end);
+ long startBegin = System.currentTimeMillis();
+ long startEnd = System.currentTimeMillis() + 1;
+ request.setStartRange(startBegin, startEnd);
+ long finishBegin = System.currentTimeMillis() + 2;
+ long finishEnd = System.currentTimeMillis() + 3;
+ request.setFinishRange(finishBegin, finishEnd);
long limit = 100L;
request.setLimit(limit);
@@ -85,11 +87,11 @@ public class TestGetApplicationsRequest
Assert.assertEquals(
"StartRange from proto is not the same with original request",
- requestFromProto.getStartRange(), new LongRange(begin, end));
+ requestFromProto.getStartRange(), new LongRange(startBegin, startEnd));
Assert.assertEquals(
"FinishRange from proto is not the same with original request",
- requestFromProto.getFinishRange(), new LongRange(begin, end));
+ requestFromProto.getFinishRange(), new LongRange(finishBegin, finishEnd));
Assert.assertEquals(
"Limit from proto is not the same with original request",
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Mon May 19 19:29:30 2014
@@ -298,9 +298,12 @@ public class ApplicationMasterService ex
List<NMToken> nmTokens = new ArrayList<NMToken>();
for (Container container : transferredContainers) {
try {
- nmTokens.add(rmContext.getNMTokenSecretManager()
- .createAndGetNMToken(app.getUser(), applicationAttemptId,
- container));
+ NMToken token = rmContext.getNMTokenSecretManager()
+ .createAndGetNMToken(app.getUser(), applicationAttemptId,
+ container);
+ if (null != token) {
+ nmTokens.add(token);
+ }
} catch (IllegalArgumentException e) {
// if it's a DNS issue, throw UnknowHostException directly and that
// will be automatically retried by RMProxy in RPC layer.
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Mon May 19 19:29:30 2014
@@ -264,31 +264,36 @@ public class TestAMRestart {
nm2.registerNode();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- int NUM_CONTAINERS = 1;
List<Container> containers = new ArrayList<Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
List<NMToken> expectedNMTokens = new ArrayList<NMToken>();
- // am1 allocate 1 container on nm1.
+ // am1 allocate 2 container on nm1.
+ // first container
while (true) {
AllocateResponse response =
- am1.allocate("127.0.0.1", 2000, NUM_CONTAINERS,
+ am1.allocate("127.0.0.1", 2000, 2,
new ArrayList<ContainerId>());
nm1.nodeHeartbeat(true);
containers.addAll(response.getAllocatedContainers());
expectedNMTokens.addAll(response.getNMTokens());
- if (containers.size() == NUM_CONTAINERS) {
+ if (containers.size() == 2) {
break;
}
Thread.sleep(200);
System.out.println("Waiting for container to be allocated.");
}
- // launch the container
+ // launch the container-2
nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
ContainerId containerId2 =
ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
-
+ // launch the container-3
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 3, ContainerState.RUNNING);
+ ContainerId containerId3 =
+ ContainerId.newInstance(am1.getApplicationAttemptId(), 3);
+ rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
+
// fail am1
nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
am1.waitForState(RMAppAttemptState.FAILED);
@@ -308,12 +313,12 @@ public class TestAMRestart {
containers = new ArrayList<Container>();
while (true) {
AllocateResponse allocateResponse =
- am2.allocate("127.1.1.1", 4000, NUM_CONTAINERS,
+ am2.allocate("127.1.1.1", 4000, 1,
new ArrayList<ContainerId>());
nm2.nodeHeartbeat(true);
containers.addAll(allocateResponse.getAllocatedContainers());
expectedNMTokens.addAll(allocateResponse.getNMTokens());
- if (containers.size() == NUM_CONTAINERS) {
+ if (containers.size() == 1) {
break;
}
Thread.sleep(200);