You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/08/31 15:05:24 UTC
[04/19] hadoop git commit: YARN-5221. Expose UpdateResourceRequest
API to allow AM to request for change in container properties. (asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index b271b37..38ef59b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2092,7 +2092,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
String nodeLabelExpression) {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
NMContainerStatus containerReport =
- NMContainerStatus.newInstance(containerId, containerState,
+ NMContainerStatus.newInstance(containerId, 0, containerState,
Resource.newInstance(1024, 1), "recover container", 0,
Priority.newInstance(0), 0, nodeLabelExpression);
return containerReport;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index aa5b336..c2a20a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -1127,7 +1127,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
NMContainerStatus report =
NMContainerStatus.newInstance(
ContainerId.newContainerId(
- ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1),
+ ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), 0,
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
rm.getResourceTrackerService().handleNMContainerStatus(report, null);
@@ -1138,7 +1138,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
(RMAppAttemptImpl) app.getCurrentAppAttempt();
currentAttempt.setMasterContainer(null);
report = NMContainerStatus.newInstance(
- ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0),
+ ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), 0,
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
rm.getResourceTrackerService().handleNMContainerStatus(report, null);
@@ -1150,7 +1150,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
// Case 2.1: AppAttemptId is null
report = NMContainerStatus.newInstance(
ContainerId.newContainerId(
- ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1),
+ ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), 0,
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
try {
@@ -1165,7 +1165,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
(RMAppAttemptImpl) app.getCurrentAppAttempt();
currentAttempt.setMasterContainer(null);
report = NMContainerStatus.newInstance(
- ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0),
+ ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), 0,
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 09c16d0..bc92c01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -58,9 +58,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -3232,9 +3233,10 @@ public class TestCapacityScheduler {
// am1 asks to change its AM container from 1GB to 3GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(3 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null)));
FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
@@ -3246,11 +3248,14 @@ public class TestCapacityScheduler {
// am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId2, Resources.createResource(3 * GB)),
- ContainerResourceChangeRequest
- .newInstance(containerId3, Resources.createResource(5 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null),
+ UpdateContainerRequest
+ .newInstance(0, containerId3,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(5 * GB), null)));
Assert.assertEquals(6 * GB,
app.getAppAttemptResourceUsage().getPending().getMemorySize());
@@ -3261,13 +3266,18 @@ public class TestCapacityScheduler {
// am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
// containerId3 (2G -> 2G)
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(3 * GB)),
- ContainerResourceChangeRequest
- .newInstance(containerId2, Resources.createResource(4 * GB)),
- ContainerResourceChangeRequest
- .newInstance(containerId3, Resources.createResource(2 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null),
+ UpdateContainerRequest
+ .newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(4 * GB), null),
+ UpdateContainerRequest
+ .newInstance(0, containerId3,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(2 * GB), null)));
Assert.assertEquals(4 * GB,
app.getAppAttemptResourceUsage().getPending().getMemorySize());
checkPendingResource(rm, "a1", 4 * GB, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 6cf9c61..e2b4952 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -289,13 +289,14 @@ public class TestContainerAllocation {
@Override
public Token createContainerToken(ContainerId containerId,
- NodeId nodeId, String appSubmitter, Resource capability,
- Priority priority, long createTime,
- LogAggregationContext logAggregationContext, String nodeLabelExp, ContainerType containerType) {
+ int containerVersion, NodeId nodeId, String appSubmitter,
+ Resource capability, Priority priority, long createTime,
+ LogAggregationContext logAggregationContext, String nodeLabelExp,
+ ContainerType containerType) {
numRetries++;
- return super.createContainerToken(containerId, nodeId, appSubmitter,
- capability, priority, createTime, logAggregationContext,
- nodeLabelExp, containerType);
+ return super.createContainerToken(containerId, containerVersion,
+ nodeId, appSubmitter, capability, priority, createTime,
+ logAggregationContext, nodeLabelExp, containerType);
}
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 499e041..6fba22a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -30,10 +30,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -84,7 +86,7 @@ public class TestContainerResizing {
@Override
protected void decreaseContainers(
- List<ContainerResourceChangeRequest> decreaseRequests,
+ List<UpdateContainerRequest> decreaseRequests,
SchedulerApplicationAttempt attempt) {
try {
Thread.sleep(1000);
@@ -138,9 +140,10 @@ public class TestContainerResizing {
sentRMContainerLaunched(rm1, containerId1);
// am1 asks to change its AM container from 1GB to 3GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(3 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null)));
FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp(
rm1, app1.getApplicationId());
@@ -195,9 +198,11 @@ public class TestContainerResizing {
sentRMContainerLaunched(rm1, containerId1);
// am1 asks to change its AM container from 1GB to 3GB
- AllocateResponse response = am1.sendContainerResizingRequest(null, Arrays
- .asList(ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(1 * GB))));
+ AllocateResponse response = am1.sendContainerResizingRequest(Arrays
+ .asList(UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.DECREASE_RESOURCE,
+ Resources.createResource(1 * GB), null)));
verifyContainerDecreased(response, containerId1, 1 * GB);
checkUsedResource(rm1, "default", 1 * GB, null);
@@ -266,9 +271,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 1GB to 3GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(7 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(7 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -367,9 +373,10 @@ public class TestContainerResizing {
// am1 asks to change container2 from 2GB to 8GB, which will exceed user
// limit
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId2, Resources.createResource(8 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(8 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -447,9 +454,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 1GB to 3GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(7 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(7 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -487,9 +495,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 1G to 1G (cancel the increase
// request actually)
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(1 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(1 * GB), null)));
// Trigger a node heartbeat..
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
@@ -559,9 +568,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 2GB to 8GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(8 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(8 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -597,9 +607,11 @@ public class TestContainerResizing {
// request, make target_capacity=existing_capacity)
am1.allocate(null, Arrays.asList(containerId2));
// am1 asks to change its AM container from 2G to 1G (decrease)
- am1.sendContainerResizingRequest(null, Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(1 * GB))));
+ am1.sendContainerResizingRequest(Arrays.asList(
+ UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(1 * GB), null)));
// Trigger a node heartbeat..
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
@@ -668,9 +680,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 2GB to 8GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId2, Resources.createResource(8 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(8 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -765,9 +778,10 @@ public class TestContainerResizing {
// am1 asks to change its AM container from 2GB to 8GB
am1.sendContainerResizingRequest(Arrays.asList(
- ContainerResourceChangeRequest
- .newInstance(containerId2, Resources.createResource(8 * GB))),
- null);
+ UpdateContainerRequest
+ .newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(8 * GB), null)));
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -883,14 +897,16 @@ public class TestContainerResizing {
allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 4, 6);
// am1 asks to change its container[2-7] from 1G to 2G
- List<ContainerResourceChangeRequest> increaseRequests = new ArrayList<>();
+ List<UpdateContainerRequest> increaseRequests = new ArrayList<>();
for (int cId = 2; cId <= 7; cId++) {
ContainerId containerId =
ContainerId.newContainerId(am1.getApplicationAttemptId(), cId);
- increaseRequests.add(ContainerResourceChangeRequest
- .newInstance(containerId, Resources.createResource(2 * GB)));
+ increaseRequests.add(UpdateContainerRequest
+ .newInstance(0, containerId,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(2 * GB), null));
}
- am1.sendContainerResizingRequest(increaseRequests, null);
+ am1.sendContainerResizingRequest(increaseRequests);
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -904,7 +920,7 @@ public class TestContainerResizing {
// earlier allocated)
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
AllocateResponse allocateResponse = am1.allocate(null, null);
- Assert.assertEquals(3, allocateResponse.getIncreasedContainers().size());
+ Assert.assertEquals(3, allocateResponse.getUpdatedContainers().size());
verifyContainerIncreased(allocateResponse,
ContainerId.newContainerId(attemptId, 4), 2 * GB);
verifyContainerIncreased(allocateResponse,
@@ -964,14 +980,16 @@ public class TestContainerResizing {
allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 4, 6);
// am1 asks to change its container[2-7] from 1G to 2G
- List<ContainerResourceChangeRequest> increaseRequests = new ArrayList<>();
+ List<UpdateContainerRequest> increaseRequests = new ArrayList<>();
for (int cId = 2; cId <= 7; cId++) {
ContainerId containerId =
ContainerId.newContainerId(am1.getApplicationAttemptId(), cId);
- increaseRequests.add(ContainerResourceChangeRequest
- .newInstance(containerId, Resources.createResource(2 * GB)));
+ increaseRequests.add(UpdateContainerRequest
+ .newInstance(0, containerId,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(2 * GB), null));
}
- am1.sendContainerResizingRequest(increaseRequests, null);
+ am1.sendContainerResizingRequest(increaseRequests);
checkPendingResource(rm1, "default", 6 * GB, null);
Assert.assertEquals(6 * GB,
@@ -985,7 +1003,7 @@ public class TestContainerResizing {
// earlier allocated)
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
AllocateResponse allocateResponse = am1.allocate(null, null);
- Assert.assertEquals(3, allocateResponse.getIncreasedContainers().size());
+ Assert.assertEquals(3, allocateResponse.getUpdatedContainers().size());
verifyContainerIncreased(allocateResponse,
ContainerId.newContainerId(attemptId, 4), 2 * GB);
verifyContainerIncreased(allocateResponse,
@@ -1046,9 +1064,11 @@ public class TestContainerResizing {
nm.nodeHeartbeat(true);
// *In the mean time*, am1 asks to decrease its AM container resource from
// 3GB to 1GB
- AllocateResponse response = am1.sendContainerResizingRequest(null,
- Collections.singletonList(ContainerResourceChangeRequest
- .newInstance(containerId1, Resources.createResource(GB))));
+ AllocateResponse response = am1.sendContainerResizingRequest(
+ Collections.singletonList(UpdateContainerRequest
+ .newInstance(0, containerId1,
+ ContainerUpdateType.DECREASE_RESOURCE,
+ Resources.createResource(GB), null)));
// verify that the containe resource is decreased
verifyContainerDecreased(response, containerId1, GB);
@@ -1077,12 +1097,16 @@ public class TestContainerResizing {
private void verifyContainerIncreased(AllocateResponse response,
ContainerId containerId, int mem) {
- List<Container> increasedContainers = response.getIncreasedContainers();
+ List<UpdatedContainer> increasedContainers =
+ response.getUpdatedContainers();
boolean found = false;
- for (Container c : increasedContainers) {
- if (c.getId().equals(containerId)) {
+ for (UpdatedContainer c : increasedContainers) {
+ if (c.getContainer().getId().equals(containerId)) {
found = true;
- Assert.assertEquals(mem, c.getResource().getMemorySize());
+ Assert.assertEquals(ContainerUpdateType.INCREASE_RESOURCE,
+ c.getUpdateType());
+ Assert.assertEquals(mem,
+ c.getContainer().getResource().getMemorySize());
}
}
if (!found) {
@@ -1092,12 +1116,16 @@ public class TestContainerResizing {
private void verifyContainerDecreased(AllocateResponse response,
ContainerId containerId, int mem) {
- List<Container> decreasedContainers = response.getDecreasedContainers();
+ List<UpdatedContainer> decreasedContainers =
+ response.getUpdatedContainers();
boolean found = false;
- for (Container c : decreasedContainers) {
- if (c.getId().equals(containerId)) {
+ for (UpdatedContainer c : decreasedContainers) {
+ if (c.getContainer().getId().equals(containerId)) {
found = true;
- Assert.assertEquals(mem, c.getResource().getMemorySize());
+ Assert.assertEquals(ContainerUpdateType.DECREASE_RESOURCE,
+ c.getUpdateType());
+ Assert.assertEquals(mem,
+ c.getContainer().getResource().getMemorySize());
}
}
if (!found) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index d388172..c5829cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -21,10 +21,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -109,8 +111,9 @@ public class TestIncreaseAllocationExpirer {
rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
// am1 asks to increase containerId2 from 1GB to 3GB
am1.sendContainerResizingRequest(Collections.singletonList(
- ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(3 * GB))), null);
+ UpdateContainerRequest.newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null)));
// Kick off scheduling and sleep for 1 second;
nm1.nodeHeartbeat(true);
Thread.sleep(1000);
@@ -180,8 +183,9 @@ public class TestIncreaseAllocationExpirer {
rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
// am1 asks to increase containerId2 from 1GB to 3GB
am1.sendContainerResizingRequest(Collections.singletonList(
- ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(3 * GB))), null);
+ UpdateContainerRequest.newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null)));
// Kick off scheduling and wait for 1 second;
nm1.nodeHeartbeat(true);
Thread.sleep(1000);
@@ -249,8 +253,9 @@ public class TestIncreaseAllocationExpirer {
rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
// am1 asks to change containerId2 from 1GB to 3GB
am1.sendContainerResizingRequest(Collections.singletonList(
- ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(3 * GB))), null);
+ UpdateContainerRequest.newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(3 * GB), null)));
// Kick off scheduling and sleep for 1 second to
// make sure the allocation is done
nm1.nodeHeartbeat(true);
@@ -261,10 +266,23 @@ public class TestIncreaseAllocationExpirer {
Resource resource1 = Resources.clone(
rm1.getResourceScheduler().getRMContainer(containerId2)
.getAllocatedResource());
+
+ // This should not work, since the container version is wrong
+ AllocateResponse response = am1.sendContainerResizingRequest(Collections
+ .singletonList(
+ UpdateContainerRequest.newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(5 * GB), null)));
+ List<UpdateContainerError> updateErrors = response.getUpdateErrors();
+ Assert.assertEquals(1, updateErrors.size());
+ Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR|0|1",
+ updateErrors.get(0).getReason());
+
// am1 asks to change containerId2 from 3GB to 5GB
am1.sendContainerResizingRequest(Collections.singletonList(
- ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(5 * GB))), null);
+ UpdateContainerRequest.newInstance(1, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(5 * GB), null)));
// Kick off scheduling and sleep for 1 second to
// make sure the allocation is done
nm1.nodeHeartbeat(true);
@@ -362,30 +380,36 @@ public class TestIncreaseAllocationExpirer {
rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
rm1.waitForState(nm1, containerId4, RMContainerState.RUNNING);
// am1 asks to change containerId2 and containerId3 from 1GB to 3GB
- List<ContainerResourceChangeRequest> increaseRequests = new ArrayList<>();
- increaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(6 * GB)));
- increaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId3, Resources.createResource(6 * GB)));
- increaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId4, Resources.createResource(6 * GB)));
- am1.sendContainerResizingRequest(increaseRequests, null);
+ List<UpdateContainerRequest> increaseRequests = new ArrayList<>();
+ increaseRequests.add(UpdateContainerRequest.newInstance(0, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(6 * GB), null));
+ increaseRequests.add(UpdateContainerRequest.newInstance(0, containerId3,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(6 * GB), null));
+ increaseRequests.add(UpdateContainerRequest.newInstance(0, containerId4,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(6 * GB), null));
+ am1.sendContainerResizingRequest(increaseRequests);
nm1.nodeHeartbeat(true);
Thread.sleep(1000);
// Start container increase allocation expirer
am1.allocate(null, null);
// Decrease containers
- List<ContainerResourceChangeRequest> decreaseRequests = new ArrayList<>();
- decreaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId2, Resources.createResource(2 * GB)));
- decreaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId3, Resources.createResource(4 * GB)));
- decreaseRequests.add(ContainerResourceChangeRequest.newInstance(
- containerId4, Resources.createResource(4 * GB)));
+ List<UpdateContainerRequest> decreaseRequests = new ArrayList<>();
+ decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId2,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(2 * GB), null));
+ decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId3,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(4 * GB), null));
+ decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId4,
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(4 * GB), null));
AllocateResponse response =
- am1.sendContainerResizingRequest(null, decreaseRequests);
+ am1.sendContainerResizingRequest(decreaseRequests);
// Verify containers are decreased in scheduler
- Assert.assertEquals(3, response.getDecreasedContainers().size());
+ Assert.assertEquals(3, response.getUpdatedContainers().size());
// Use the token for containerId4 on NM (6G). This should set the last
// confirmed resource to 4G, and cancel the allocation expirer
nm1.containerIncreaseStatus(getContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 59bb6aa..ee3396d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -220,7 +220,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
org.apache.hadoop.yarn.api.records.Token validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r, Priority.newInstance(10), 1234);
+ 0, validNode, user, r, Priority.newInstance(10), 1234);
ContainerTokenIdentifier identifier =
BuilderUtils.newContainerTokenIdentifier(validContainerToken);
Assert.assertEquals(Priority.newInstance(10), identifier.getPriority());
@@ -277,7 +277,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
4 * 60 * 1000);
validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r, Priority.newInstance(0), 0);
+ 0, validNode, user, r, Priority.newInstance(0), 0);
Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, validNMToken, false).isEmpty());
Assert.assertTrue(nmTokenSecretManagerNM
@@ -293,7 +293,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
org.apache.hadoop.yarn.api.records.Token validContainerToken2 =
containerTokenSecretManager.createContainerToken(validContainerId2,
- validNode, user, r, Priority.newInstance(0), 0);
+ 0, validNode, user, r, Priority.newInstance(0), 0);
org.apache.hadoop.yarn.api.records.Token validNMToken2 =
nmTokenSecretManagerRM.createNMToken(validAppAttemptId2, validNode, user);
@@ -379,7 +379,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
.createNMToken(validAppAttemptId, validNode, user);
org.apache.hadoop.yarn.api.records.Token newContainerToken =
containerTokenSecretManager.createContainerToken(
- ContainerId.newContainerId(attempt2, 1), validNode, user, r,
+ ContainerId.newContainerId(attempt2, 1), 0, validNode, user, r,
Priority.newInstance(0), 0);
Assert.assertTrue(testStartContainer(rpc, attempt2, validNode,
newContainerToken, attempt1NMToken, false).isEmpty());
@@ -639,7 +639,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
Token containerToken =
containerTokenSecretManager.createContainerToken(
- cId, nodeId, user, r, Priority.newInstance(0), 0);
+ cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier =
getContainerTokenIdentifierFromToken(containerToken);
@@ -672,8 +672,8 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
// Creating modified containerToken
Token containerToken2 =
- tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
- user, r, Priority.newInstance(0), 0);
+ tamperedContainerTokenSecretManager.createContainerToken(cId2, 0,
+ nodeId, user, r, Priority.newInstance(0), 0);
StringBuilder sb = new StringBuilder("Given Container ");
sb.append(cId2);
@@ -731,8 +731,8 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
getContainerTokenSecretManager();
Resource r = Resource.newInstance(1230, 2);
Token containerToken =
- containerTokenSecretManager.createContainerToken(cId, nodeId, user, r,
- Priority.newInstance(0), 0);
+ containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user,
+ r, Priority.newInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
index 49a82c8..a941302 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
@@ -198,8 +198,6 @@ public class TestMiniYarnClusterNodeUtilization {
/**
* Verify both the RMNode and SchedulerNode have been updated with the test
* fixture utilization data.
- * @param containersUtilization Utilization of the container.
- * @param nodeUtilization Utilization of the node.
*/
private void verifySimulatedUtilization() throws InterruptedException {
ResourceManager rm = cluster.getResourceManager(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d9cff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/proto/test_token.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/proto/test_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/proto/test_token.proto
index 853f477..c111462 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/proto/test_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/proto/test_token.proto
@@ -44,5 +44,6 @@ message ContainerTokenIdentifierForTestProto {
optional int64 creationTime = 9;
optional LogAggregationContextProto logAggregationContext = 10;
optional string message = 11;
+ optional int32 version = 14 [default = 0];
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org