You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/12/30 01:38:30 UTC
[46/49] hadoop git commit: YARN-7669. API and interface modifications
for placement constraint processor. (asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
new file mode 100644
index 0000000..9571f0e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm
+ * is free to produce multiple of output objects at the end of each run and it
+ * must use the provided ConstraintPlacementAlgorithmOutputCollector to
+ * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer
+ * which is provided a collector to collect output.
+ */
+public class ConstraintPlacementAlgorithmOutput {
+
+ private final ApplicationId applicationId;
+
+ public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ private final List<PlacedSchedulingRequest> placedRequests =
+ new ArrayList<>();
+
+ private final List<SchedulingRequest> rejectedRequests =
+ new ArrayList<>();
+
+ public List<PlacedSchedulingRequest> getPlacedRequests() {
+ return placedRequests;
+ }
+
+ public List<SchedulingRequest> getRejectedRequests() {
+ return rejectedRequests;
+ }
+
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
new file mode 100644
index 0000000..131fd42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+/**
+ * The ConstraintPlacementAlgorithm uses the
+ * ConstraintPlacementAlgorithmOutputCollector to collect any output it
+ * spits out.
+ */
+public interface ConstraintPlacementAlgorithmOutputCollector {
+
+ /**
+ * Collect an ConstraintPlacementAlgorithm output.
+ * @param algorithmOutput ConstraintPlacementAlgorithm Output.
+ */
+ void collect(ConstraintPlacementAlgorithmOutput algorithmOutput);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
new file mode 100644
index 0000000..2cd90d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Class to encapsulate a Placed scheduling Request.
+ * It has the original Scheduling Request and a list of SchedulerNodes (one
+ * for each 'numAllocation' field in the corresponding ResourceSizing object.
+ *
+ * NOTE: Clients of this class SHOULD NOT rely on the value of
+ * resourceSizing.numAllocations and instead should use the
+ * size of collection returned by getNodes() instead.
+ */
+public class PlacedSchedulingRequest {
+
+ // The number of times the Algorithm tried to place the SchedulingRequest
+ // after it was rejected by the commit phase of the Scheduler (due to some
+ // transient state of the cluster. For eg: no space on Node / user limit etc.)
+ // The Algorithm can then try to probably place on a different node.
+ private int placementAttempt = 0;
+ private final SchedulingRequest request;
+ // One Node per numContainers in the SchedulingRequest;
+ private final List<SchedulerNode> nodes = new ArrayList<>();
+
+ public PlacedSchedulingRequest(SchedulingRequest request) {
+ this.request = request;
+ }
+
+ public SchedulingRequest getSchedulingRequest() {
+ return request;
+ }
+
+ /**
+ * List of Node locations on which this Scheduling Request can be placed.
+ * The size of this list = schedulingRequest.resourceSizing.numAllocations.
+ * @return List of Scheduler nodes.
+ */
+ public List<SchedulerNode> getNodes() {
+ return nodes;
+ }
+
+ public int getPlacementAttempt() {
+ return placementAttempt;
+ }
+
+ public void setPlacementAttempt(int attempt) {
+ this.placementAttempt = attempt;
+ }
+
+ @Override
+ public String toString() {
+ return "PlacedSchedulingRequest{" +
+ "placementAttempt=" + placementAttempt +
+ ", request=" + request +
+ ", nodes=" + nodes +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
new file mode 100644
index 0000000..6c65d84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+/**
+ * This class encapsulates the response received from the ResourceScheduler's
+ * attemptAllocateOnNode method.
+ */
+public class SchedulingResponse {
+
+ private final boolean isSuccess;
+ private final ApplicationId applicationId;
+ private final SchedulingRequest schedulingRequest;
+
+ /**
+ * Create a SchedulingResponse.
+ * @param isSuccess did scheduler accept.
+ * @param applicationId Application Id.
+ * @param schedulingRequest Scheduling Request.
+ */
+ public SchedulingResponse(boolean isSuccess, ApplicationId applicationId,
+ SchedulingRequest schedulingRequest) {
+ this.isSuccess = isSuccess;
+ this.applicationId = applicationId;
+ this.schedulingRequest = schedulingRequest;
+ }
+
+ /**
+ * Returns true if Scheduler was able to accept and commit this request.
+ * @return isSuccessful.
+ */
+ public boolean isSuccess() {
+ return this.isSuccess;
+ }
+
+ /**
+ * Get Application Id.
+ * @return Application Id.
+ */
+ public ApplicationId getApplicationId() {
+ return this.applicationId;
+ }
+
+ /**
+ * Get Scheduling Request.
+ * @return Scheduling Request.
+ */
+ public SchedulingRequest getSchedulingRequest() {
+ return this.schedulingRequest;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
new file mode 100644
index 0000000..01ed713
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This Package contains classes related to constrained placement of
+ * Requests.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
deleted file mode 100644
index 0358792..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * /
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.constraint;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test functionality of AllocationTagsManager.
- */
-public class TestAllocationTagsManager {
- @Test
- public void testAllocationTagsManagerSimpleCases()
- throws InvalidAllocationTagsQueryException {
- AllocationTagsManager atm = new AllocationTagsManager();
-
- /**
- * Construct test case:
- * Node1:
- * container_1_1 (mapper/reducer/app_1)
- * container_1_3 (service/app_1)
- *
- * Node2:
- * container_1_2 (mapper/reducer/app_1)
- * container_1_4 (reducer/app_1)
- * container_2_1 (service/app_2)
- */
-
- // 3 Containers from app1
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- // 1 Container from app2
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Get Cardinality of app1 on node1, with tag "mapper"
- Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
- Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
- Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
- Assert.assertEquals(3,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::sum));
-
- // Get Cardinality by passing single tag.
- Assert.assertEquals(1,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), "mapper"));
-
- Assert.assertEquals(2,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), "reducer"));
-
- // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("no_existed", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
- // (Expect this returns #containers from app1 on node2)
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet
- .of(AllocationTagsNamespaces.APP_ID + TestUtils
- .getMockApplicationId(1).toString()), Long::max));
-
- // Get Cardinality of app1 on node2, with empty tag set, op=max
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
-
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(7,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(5,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
-
- // Finish all containers:
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Expect all cardinality to be 0
- // Get Cardinality of app1 on node1, with tag "mapper"
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
- Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::sum));
-
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
- // (Expect this returns #containers from app1 on node2)
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()),
- Long::max));
-
- Assert.assertEquals(0,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- TestUtils.getMockApplicationId(1).toString()));
-
- // Get Cardinality of app1 on node2, with empty tag set, op=max
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
-
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
- }
-
- @Test
- public void testAllocationTagsManagerMemoryAfterCleanup()
- throws InvalidAllocationTagsQueryException {
- /**
- * Make sure YARN cleans up all memory once container/app finishes.
- */
-
- AllocationTagsManager atm = new AllocationTagsManager();
-
- // Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Remove all these containers
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Check internal data structure
- Assert.assertEquals(0,
- atm.getGlobalMapping().getNodeToTagsWithCount().size());
- Assert.assertEquals(0, atm.getPerAppMappings().size());
- }
-
- @Test
- public void testQueryCardinalityWithIllegalParameters()
- throws InvalidAllocationTagsQueryException {
- /**
- * Make sure YARN cleans up all memory once container/app finishes.
- */
-
- AllocationTagsManager atm = new AllocationTagsManager();
-
- // Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // No node-id
- boolean caughtException = false;
- try {
- atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2),
- ImmutableSet.of("mapper"), Long::min);
- } catch (InvalidAllocationTagsQueryException e) {
- caughtException = true;
- }
- Assert.assertTrue("should fail because of nodeId specified",
- caughtException);
-
- // No op
- caughtException = false;
- try {
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null);
- } catch (InvalidAllocationTagsQueryException e) {
- caughtException = true;
- }
- Assert.assertTrue("should fail because of nodeId specified",
- caughtException);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 27ff311..538d128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -62,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 61a5555..e8734cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -42,12 +42,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
new file mode 100644
index 0000000..4bb2a18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
@@ -0,0 +1,328 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test functionality of AllocationTagsManager.
+ */
+public class TestAllocationTagsManager {
+ @Test
+ public void testAllocationTagsManagerSimpleCases()
+ throws InvalidAllocationTagsQueryException {
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ /**
+ * Construct test case:
+ * Node1:
+ * container_1_1 (mapper/reducer/app_1)
+ * container_1_3 (service/app_1)
+ *
+ * Node2:
+ * container_1_2 (mapper/reducer/app_1)
+ * container_1_4 (reducer/app_1)
+ * container_2_1 (service/app_2)
+ */
+
+ // 3 Containers from app1
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ // 1 Container from app2
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Get Cardinality of app1 on node1, with tag "mapper"
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ Assert.assertEquals(3,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::sum));
+
+ // Get Cardinality by passing single tag.
+ Assert.assertEquals(1,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), "mapper"));
+
+ Assert.assertEquals(2,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), "reducer"));
+
+ // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("no_existed", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // (Expect this returns #containers from app1 on node2)
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet
+ .of(AllocationTagsNamespaces.APP_ID + TestUtils
+ .getMockApplicationId(1).toString()), Long::max));
+
+ // Get Cardinality of app1 on node2, with empty tag set, op=max
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
+
+ // Get Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(7,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
+ ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(5,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
+
+ // Finish all containers:
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Expect all cardinality to be 0
+ // Get Cardinality of app1 on node1, with tag "mapper"
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::sum));
+
+ // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // (Expect this returns #containers from app1 on node2)
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()),
+ Long::max));
+
+ Assert.assertEquals(0,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ TestUtils.getMockApplicationId(1).toString()));
+
+ // Get Cardinality of app1 on node2, with empty tag set, op=max
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
+
+ // Get Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
+ ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
+ }
+
+ @Test
+ public void testAllocationTagsManagerMemoryAfterCleanup()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Make sure YARN cleans up all memory once container/app finishes.
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ // Add a bunch of containers
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Remove all these containers
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Check internal data structure
+ Assert.assertEquals(0,
+ atm.getGlobalMapping().getNodeToTagsWithCount().size());
+ Assert.assertEquals(0, atm.getPerAppMappings().size());
+ }
+
+ @Test
+ public void testQueryCardinalityWithIllegalParameters()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Make sure YARN cleans up all memory once container/app finishes.
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ // Add a bunch of containers
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // No node-id
+ boolean caughtException = false;
+ try {
+ atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2),
+ ImmutableSet.of("mapper"), Long::min);
+ } catch (InvalidAllocationTagsQueryException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue("should fail because of nodeId specified",
+ caughtException);
+
+ // No op
+ caughtException = false;
+ try {
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null);
+ } catch (InvalidAllocationTagsQueryException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue("should fail because of nodeId specified",
+ caughtException);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6062844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 4b902a7..db749ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -74,7 +74,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org