You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:09 UTC
[27/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
new file mode 100644
index 0000000..b0634bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.NodeMap;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test that app state is rebuilt on a restart.
+ */
+public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ public String getTestName() {
+ return "TestMockAppStateRebuildOnAMRestart";
+ }
+
+ @Test
+ public void testRebuild() throws Throwable {
+
+ int r0 = 1;
+ int r1 = 2;
+ int r2 = 3;
+ getRole0Status().setDesired(r0);
+ getRole1Status().setDesired(r1);
+ getRole2Status().setDesired(r2);
+ List<RoleInstance> instances = createAndStartNodes();
+
+ int clusterSize = r0 + r1 + r2;
+ assertEquals(instances.size(), clusterSize);
+
+ //clone the list
+ List<Container> containers = new ArrayList<>();
+ for (RoleInstance ri : instances) {
+ containers.add(ri.container);
+ }
+ NodeMap nodemap = appState.getRoleHistory().cloneNodemap();
+
+ //and rebuild
+
+ AppStateBindingInfo bindingInfo = buildBindingInfo();
+ bindingInfo.application = factory.newApplication(r0, r1, r2)
+ .name(getTestName());
+ bindingInfo.liveContainers = containers;
+ appState = new MockAppState(bindingInfo);
+
+ assertEquals(appState.getLiveContainers().size(), clusterSize);
+
+ appState.getRoleHistory().dump();
+
+ //check that the app state direct structures match
+ List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0);
+ List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1);
+ List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2);
+
+ assertEquals(r0, r0live.size());
+ assertEquals(r1, r1live.size());
+ assertEquals(r2, r2live.size());
+
+ //now examine the role history
+ NodeMap newNodemap = appState.getRoleHistory().cloneNodemap();
+
+ for (NodeInstance nodeInstance : newNodemap.values()) {
+ String hostname = nodeInstance.hostname;
+ NodeInstance orig = nodemap.get(hostname);
+ assertNotNull("Null entry in original nodemap for " + hostname, orig);
+
+ for (int i : Arrays.asList(getRole0Status().getKey(), getRole1Status()
+ .getKey(), getRole2Status().getKey())) {
+ assertEquals(nodeInstance.getActiveRoleInstances(i), orig
+ .getActiveRoleInstances(i));
+ NodeEntry origRE = orig.getOrCreate(i);
+ NodeEntry newRE = nodeInstance.getOrCreate(i);
+ assertEquals(origRE.getLive(), newRE.getLive());
+ assertEquals(0, newRE.getStarting());
+ }
+ }
+ assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
+
+ Application application = appState.getClusterStatus();
+ // verify the AM restart container count was set
+ Long restarted = application.getNumberOfRunningContainers();
+ assertNotNull(restarted);
+ //and that the count == 1 master + the region servers
+ assertEquals(restarted.longValue(), (long)containers.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
new file mode 100644
index 0000000..946f1c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.RoleHistoryUtils;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole;
+
+/**
+ * Test that the app state lets you ask for nodes, get a specific host,
+ * release it and then get that one back again.
+ */
+public class TestMockAppStateRolePlacement extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ public String getTestName() {
+ return "TestMockAppStateRolePlacement";
+ }
+
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ getRole0Status().setDesired(1);
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ ContainerRequestOperation operation = (ContainerRequestOperation)ops
+ .get(0);
+ AMRMClient.ContainerRequest request = operation.getRequest();
+ assertTrue(request.getRelaxLocality());
+ assertNull(request.getNodes());
+ assertNull(request.getRacks());
+ assertNotNull(request.getCapability());
+
+ Container allocated = engine.allocateContainer(request);
+ List<ContainerAssignment> assignments = new ArrayList<>();
+ List<AbstractRMOperation> releaseOperations = new ArrayList<>();
+ appState.onContainersAllocated(Arrays.asList((Container)allocated),
+ assignments, releaseOperations);
+ // verify the release matches the allocation
+ assertEquals(releaseOperations.size(), 1);
+ CancelSingleRequest cancelOp = (CancelSingleRequest)releaseOperations
+ .get(0);
+ assertNotNull(cancelOp.getRequest());
+ assertNotNull(cancelOp.getRequest().getCapability());
+ assertEquals(cancelOp.getRequest().getCapability(), allocated
+ .getResource());
+ // now the assignment
+ assertEquals(assignments.size(), 1);
+ ContainerAssignment assigned = assignments.get(0);
+ Container container = assigned.container;
+ assertEquals(container.getId(), allocated.getId());
+ int roleId = assigned.role.getPriority();
+ assertEquals(roleId, extractRole(request.getPriority()));
+ assertEquals(assigned.role.getName(), ROLE0);
+ String containerHostname = RoleHistoryUtils.hostnameOf(container);
+ RoleInstance ri = roleInstance(assigned);
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ assertNotNull(appState.onNodeManagerContainerStarted(container.getId()));
+ assertEquals(getRole0Status().getRunning(), 1);
+ ops = appState.reviewRequestAndReleaseNodes();
+ assertEquals(ops.size(), 0);
+
+ //now it is surplus
+ getRole0Status().setDesired(0);
+ ops = appState.reviewRequestAndReleaseNodes();
+ ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0);
+
+ assertEquals(release.getContainerId(), container.getId());
+ engine.execute(ops);
+ assertNotNull(appState.onCompletedContainer(containerStatus(container
+ .getId())).roleInstance);
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ //now ask for a new one
+ getRole0Status().setDesired(1);
+ ops = appState.reviewRequestAndReleaseNodes();
+ assertEquals(ops.size(), 1);
+ operation = (ContainerRequestOperation) ops.get(0);
+ AMRMClient.ContainerRequest request2 = operation.getRequest();
+ assertNotNull(request2);
+ assertEquals(request2.getNodes().get(0), containerHostname);
+ assertFalse(request2.getRelaxLocality());
+ engine.execute(ops);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
new file mode 100644
index 0000000..d62a91e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+public class TestMockAppStateRoleRelease extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ public String getTestName() {
+ return "TestMockAppStateRoleRelease";
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node.
+ * @return
+ */
+ @Override
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4);
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ /**
+ * Allocate to all nodes
+ */
+ getRole0Status().setDesired(6);
+ getRole1Status().setDesired(5);
+ getRole2Status().setDesired(4);
+ List<RoleInstance> instances = createAndStartNodes();
+ assertEquals(instances.size(), 15);
+
+ //now it is surplus
+ getRole0Status().setDesired(0);
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+
+ List<ContainerId> released = new ArrayList<>();
+ engine.execute(ops, released);
+ List<ContainerId> ids = extractContainerIds(instances, ROLE0);
+ for (ContainerId cid : released) {
+ assertNotNull(appState.onCompletedContainer(containerStatus(cid))
+ .roleInstance);
+ assertTrue(ids.contains(cid));
+ }
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
new file mode 100644
index 0000000..eaf5271
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+
+import java.util.Collections;
+
+/**
+ * Test that if you have more than one role, the right roles are chosen for
+ * release.
+ */
+public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ public String getTestName() {
+ return "TestMockAppStateUniqueNames";
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node.
+ * @return
+ */
+ @Override
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4);
+ }
+
+ @Override
+ public AppStateBindingInfo buildBindingInfo() {
+ AppStateBindingInfo bindingInfo = super.buildBindingInfo();
+ bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
+ return bindingInfo;
+ }
+
+ @Override
+ public Application buildApplication() {
+ Application application = super.buildApplication();
+
+ Component component = new Component().name("group1").numberOfContainers(2L)
+ .resource(new Resource().memory("1024").cpus(2))
+ .uniqueComponentSupport(true);
+
+ application.getComponents().add(component);
+ return application;
+ }
+
+ @Test
+ public void testDynamicFlexDown() throws Throwable {
+ createAndStartNodes();
+ appState.updateComponents(Collections.singletonMap("group1", 0L));
+ createAndStartNodes();
+ RoleStatus roleStatus = appState.lookupRoleStatus("group11");
+ assertEquals(0, roleStatus.getDesired());
+ assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize());
+ assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores());
+ assertEquals("group1", roleStatus.getGroup());
+ }
+
+ @Test
+ public void testDynamicFlexUp() throws Throwable {
+ createAndStartNodes();
+ appState.updateComponents(Collections.singletonMap("group1", 3L));
+ createAndStartNodes();
+ RoleStatus group11 = appState.lookupRoleStatus("group11");
+ RoleStatus group12 = appState.lookupRoleStatus("group12");
+ RoleStatus group13 = appState.lookupRoleStatus("group13");
+ assertEquals(1, group11.getDesired());
+ assertEquals(1, group12.getDesired());
+ assertEquals(1, group13.getDesired());
+ assertEquals(1024L, group11.getResourceRequirements().getMemorySize());
+ assertEquals(1024L, group12.getResourceRequirements().getMemorySize());
+ assertEquals(1024L, group13.getResourceRequirements().getMemorySize());
+ assertEquals(2, group11.getResourceRequirements().getVirtualCores());
+ assertEquals(2, group12.getResourceRequirements().getVirtualCores());
+ assertEquals(2, group13.getResourceRequirements().getVirtualCores());
+ assertEquals("group1", group11.getGroup());
+ assertEquals("group1", group12.getGroup());
+ assertEquals("group1", group13.getGroup());
+
+ appState.refreshClusterStatus();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
new file mode 100644
index 0000000..046bd83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Test the container resource allocation logic.
+ */
+public class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
+
+ @Override
+ public Application buildApplication() {
+ return factory.newApplication(1, 0, 0).name(getTestName());
+ }
+
+ @Test
+ public void testNormalAllocations() throws Throwable {
+ Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
+ role0.resource(new org.apache.slider.api.resource.Resource().memory("512")
+ .cpus(2));
+ appState.updateComponents(Collections.singletonMap(role0.getName(),
+ role0.getNumberOfContainers()));
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ assertEquals(1, ops.size());
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops
+ .get(0);
+ Resource requirements = operation.getRequest().getCapability();
+ assertEquals(512L, requirements.getMemorySize());
+ assertEquals(2, requirements.getVirtualCores());
+ }
+
+ @Test
+ public void testMaxMemAllocations() throws Throwable {
+ // max core allocations no longer supported
+ Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
+ role0.resource(new org.apache.slider.api.resource.Resource()
+ .memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2));
+ appState.updateComponents(Collections.singletonMap(role0.getName(),
+ role0.getNumberOfContainers()));
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ assertEquals(1, ops.size());
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops
+ .get(0);
+ Resource requirements = operation.getRequest().getCapability();
+ assertEquals(MockAppState.RM_MAX_RAM, requirements.getMemorySize());
+ assertEquals(2, requirements.getVirtualCores());
+ }
+
+ @Test
+ public void testMaxDefaultAllocations() throws Throwable {
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ assertEquals(ops.size(), 1);
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops
+ .get(0);
+ Resource requirements = operation.getRequest().getCapability();
+ assertEquals(ResourceKeys.DEF_YARN_MEMORY, requirements.getMemorySize());
+ assertEquals(ResourceKeys.DEF_YARN_CORES, requirements.getVirtualCores());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
new file mode 100644
index 0000000..a3f8abd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test Anti-affine placement.
+ */
+public class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
+ implements MockRoles {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMockLabelledAAPlacement.class);
+
+ private static final int NODES = 3;
+ private static final int GPU_NODES = 2;
+ private static final String HOST0 = "00000000";
+ private static final String HOST1 = "00000001";
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ updateNodes(MockFactory.INSTANCE.newNodeReport(HOST0, NodeState.RUNNING,
+ LABEL_GPU));
+ updateNodes(MockFactory.INSTANCE.newNodeReport(HOST1, NodeState.RUNNING,
+ LABEL_GPU));
+ }
+
+ @Override
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(NODES, 8);
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(getGpuRole().getKey());
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testAskForTooMany() throws Throwable {
+ RoleStatus gpuRole = getGpuRole();
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ " expect the final request to be unsatisfied until the cluster " +
+ "changes size");
+ //more than expected
+ int size = GPU_NODES;
+ gpuRole.setDesired(size + 1);
+
+ List<AbstractRMOperation > operations = appState
+ .reviewRequestAndReleaseNodes();
+ assertTrue(gpuRole.isAARequestOutstanding());
+
+ assertEquals(gpuRole.getAAPending(), size);
+ for (int i = 0; i < size; i++) {
+ String iter = "Iteration " + i + " role = " + getAaRole();
+ describe(iter);
+ List<AbstractRMOperation > operationsOut = new ArrayList<>();
+
+ List<RoleInstance> roleInstances = submitOperations(operations,
+ EMPTY_ID_LIST, operationsOut);
+ // one instance per request
+ assertEquals(1, roleInstances.size());
+ appState.onNodeManagerContainerStarted(roleInstances.get(0)
+ .getContainerId());
+ assertAllContainersAA();
+ // there should be none left
+ LOG.debug(nodeInformationSnapshotAsString());
+ operations = operationsOut;
+ if (i + 1 < size) {
+ assertEquals(2, operations.size());
+ } else {
+ assertEquals(1, operations.size());
+ }
+ }
+ // expect an outstanding AA request to be unsatisfied
+ assertTrue(gpuRole.getRunning() < gpuRole.getDesired());
+ assertEquals(0, gpuRole.getRequested());
+ assertFalse(gpuRole.isAARequestOutstanding());
+ List<Container> allocatedContainers = engine.execute(operations,
+ EMPTY_ID_LIST);
+ assertEquals(0, allocatedContainers.size());
+ // in a review now, no more requests can be generated, as there is no
+ // space for AA placements, even though there is cluster capacity
+ assertEquals(0, appState.reviewRequestAndReleaseNodes().size());
+
+ // switch node 2 into being labelled
+ NodeUpdatedOutcome outcome = updateNodes(MockFactory.INSTANCE.
+ newNodeReport("00000002", NodeState.RUNNING, "gpu"));
+
+ assertEquals(NODES, cloneNodemap().size());
+ assertTrue(outcome.clusterChanged);
+ // no active calls to empty
+ assertTrue(outcome.operations.isEmpty());
+ assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
+ }
+
+ protected AppState.NodeUpdatedOutcome addNewNode() {
+ return updateNodes(MockFactory.INSTANCE.newNodeReport("00000004",
+ NodeState.RUNNING, "gpu"));
+ }
+
+ @Test
+ public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
+ RoleStatus gpuRole = getGpuRole();
+ describe("Change the cluster size where the cluster size changes during " +
+ "a test sequence.");
+ gpuRole.setDesired(GPU_NODES + 1);
+ List<AbstractRMOperation> operations = appState
+ .reviewRequestAndReleaseNodes();
+ assertTrue(gpuRole.isAARequestOutstanding());
+ assertEquals(GPU_NODES, gpuRole.getAAPending());
+ NodeUpdatedOutcome outcome = addNewNode();
+ assertTrue(outcome.clusterChanged);
+ // one call to cancel
+ assertEquals(1, outcome.operations.size());
+ // and on a review, one more to rebuild
+ assertEquals(1, appState.reviewRequestAndReleaseNodes().size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
new file mode 100644
index 0000000..5ae626e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.OutstandingRequest;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+
+/**
+ * Test outstanding request validation.
+ */
+public class TestOutstandingRequestValidation extends SliderTestBase {
+
+ private static final String[] H1 = hosts("one");
+
+ @Test
+ public void testRelaxedNohostsOrLabels() throws Throwable {
+ createAndValidate(null, null, true);
+ }
+
+ @Test
+ public void testRelaxedLabels() throws Throwable {
+ createAndValidate(null, "gpu", true);
+ }
+
+ @Test
+ public void testNonRelaxedLabels() throws Throwable {
+ expectCreationFailure(null, "gpu", false);
+ }
+
+ @Test
+ public void testRelaxedHostNoLabel() throws Throwable {
+ createAndValidate(H1, "", true);
+ }
+
+ /**
+ * Use varargs for simple list to array conversion.
+ * @param hostnames host names
+ * @return
+ */
+ public static String[] hosts(String...hostnames) {
+ return hostnames;
+ }
+
+ void expectCreationFailure(
+ String[] hosts,
+ String labels,
+ boolean relaxLocality) {
+ try {
+ ContainerRequest result = createAndValidate(hosts, labels, relaxLocality);
+ fail("Expected an exception, got " + result);
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.toString()
+ .contains("Can't turn off locality relaxation on a request with no " +
+ "location constraints"));
+ }
+ }
+
+ AMRMClient.ContainerRequest createAndValidate(
+ String[] hosts,
+ String labels,
+ boolean relaxLocality) {
+ int cores = 1;
+ int memory = 64;
+ int p = 1;
+ Priority pri = ContainerPriority.createPriority(p, !relaxLocality);
+ ContainerRequest issuedRequest =
+ newRequest(pri, hosts, labels, relaxLocality);
+ OutstandingRequest.validateContainerRequest(issuedRequest, p, "");
+ return issuedRequest;
+ }
+
+ AMRMClient.ContainerRequest newRequest(
+ Priority pri,
+ String[] hosts,
+ String labels,
+ boolean relaxLocality) {
+ int cores = 1;
+ int memory = 64;
+ Resource resource = Resource.newInstance(memory, cores);
+ return new AMRMClient.ContainerRequest(resource,
+ hosts,
+ null,
+ pri,
+ relaxLocality,
+ labels);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
new file mode 100644
index 0000000..077a6d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.api.types.RestTypeMarshalling;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.NodeMap;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test anti-affine placement.
+ */
+public class TestRoleHistoryAA extends SliderTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryAA.class);
+
+ private List<String> hostnames = Arrays.asList("1", "2", "3");
+ private NodeMap nodeMap, gpuNodeMap;
+ private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+
+ public TestRoleHistoryAA() throws BadConfigException {
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ nodeMap = createNodeMap(hostnames, NodeState.RUNNING, "");
+ gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU");
+ }
+
+ @Test
+ public void testFindNodesInFullCluster() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, nodeMap.findAllNodesForRole(1, ""));
+ }
+
+ @Test
+ public void testFindNodesInUnhealthyCluster() throws Throwable {
+ // all three will surface at first
+ markNodeOneUnhealthy();
+ verifyResultSize(2, nodeMap.findAllNodesForRole(1, ""));
+ }
+
+ public boolean markNodeOneUnhealthy() {
+ return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY);
+ }
+
+ protected boolean setNodeState(NodeInstance node, NodeState state) {
+ return node.updateNode(MockFactory.INSTANCE.newNodeReport(node.hostname,
+ state, ""));
+ }
+
+ @Test
+ public void testFindNoNodesWrongLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"));
+ }
+
+ @Test
+ public void testFindSomeNodesSomeLabel() throws Throwable {
+ // all three will surface at first
+ update(nodeMap,
+ Arrays.asList(MockFactory.INSTANCE.newNodeReport("1", NodeState
+ .RUNNING, "GPU")));
+ List<NodeInstance> gpuNodes = nodeMap.findAllNodesForRole(1, "GPU");
+ verifyResultSize(1, gpuNodes);
+ NodeInstance instance = gpuNodes.get(0);
+ instance.getOrCreate(1).onStarting();
+ assertFalse(instance.canHost(1, "GPU"));
+ assertFalse(instance.canHost(1, ""));
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"));
+
+ }
+
+ @Test
+ public void testFindNoNodesRightLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"));
+ }
+
+ @Test
+ public void testFindNoNodesNoLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""));
+ }
+
+ @Test
+ public void testFindNoNodesClusterRequested() throws Throwable {
+ // all three will surface at first
+ for (NodeInstance ni : nodeMap.values()) {
+ ni.getOrCreate(1).request();
+ }
+ assertNoAvailableNodes(1);
+ }
+
+ @Test
+ public void testFindNoNodesClusterBusy() throws Throwable {
+ // all three will surface at first
+ for (NodeInstance ni : nodeMap.values()) {
+ ni.getOrCreate(1).request();
+ }
+ assertNoAvailableNodes(1);
+ }
+
+ /**
+ * Tag all nodes as starting, then walk one through a bit
+ * more of its lifecycle.
+ */
+ @Test
+ public void testFindNoNodesLifecycle() throws Throwable {
+ // all three will surface at first
+ for (NodeInstance ni : nodeMap.values()) {
+ ni.getOrCreate(1).onStarting();
+ }
+ assertNoAvailableNodes(1);
+
+ // walk one of the nodes through the lifecycle
+ NodeInstance node1 = nodeMap.get("1");
+ assertFalse(node1.canHost(1, ""));
+ node1.get(1).onStartCompleted();
+ assertFalse(node1.canHost(1, ""));
+ assertNoAvailableNodes(1);
+ node1.get(1).release();
+ assertTrue(node1.canHost(1, ""));
+ List<NodeInstance> list2 =
+ verifyResultSize(1, nodeMap.findAllNodesForRole(1, ""));
+ assertEquals(list2.get(0).hostname, "1");
+
+ // now tag that node as unhealthy and expect it to go away
+ markNodeOneUnhealthy();
+ assertNoAvailableNodes(1);
+ }
+
+ @Test
+ public void testRolesIndependent() throws Throwable {
+ NodeInstance node1 = nodeMap.get("1");
+ NodeEntry role1 = node1.getOrCreate(1);
+ NodeEntry role2 = node1.getOrCreate(2);
+ for (NodeInstance ni : nodeMap.values()) {
+ ni.updateNode(MockFactory.INSTANCE.newNodeReport("0", NodeState
+ .UNHEALTHY, ""));
+ }
+ assertNoAvailableNodes(1);
+ assertNoAvailableNodes(2);
+ assertTrue(setNodeState(node1, NodeState.RUNNING));
+ // tag role 1 as busy
+ role1.onStarting();
+ assertNoAvailableNodes(1);
+
+ verifyResultSize(1, nodeMap.findAllNodesForRole(2, ""));
+ assertTrue(node1.canHost(2, ""));
+ }
+
+ @Test
+ public void testNodeEntryAvailablity() throws Throwable {
+ NodeEntry entry = new NodeEntry(1);
+ assertTrue(entry.isAvailable());
+ entry.onStarting();
+ assertFalse(entry.isAvailable());
+ entry.onStartCompleted();
+ assertFalse(entry.isAvailable());
+ entry.release();
+ assertTrue(entry.isAvailable());
+ entry.onStarting();
+ assertFalse(entry.isAvailable());
+ entry.onStartFailed();
+ assertTrue(entry.isAvailable());
+ }
+
+ @Test
+ public void testNodeInstanceSerialization() throws Throwable {
+ MockRoleHistory rh2 = new MockRoleHistory(new ArrayList<>());
+ rh2.getOrCreateNodeInstance("localhost");
+ NodeInstance instance = rh2.getOrCreateNodeInstance("localhost");
+ instance.getOrCreate(1).onStartCompleted();
+ Map<Integer, String> naming = Collections.singletonMap(1, "manager");
+ NodeInformation ni = instance.serialize(naming);
+ assertEquals(1, ni.entries.get("manager").live);
+ NodeInformation ni2 = rh2.getNodeInformation("localhost", naming);
+ assertEquals(1, ni2.entries.get("manager").live);
+ Map<String, NodeInformation> info = rh2.getNodeInformationSnapshot(naming);
+ assertEquals(1, info.get("localhost").entries.get("manager").live);
+ NodeInformationList nil = new NodeInformationList(info.values());
+ assertEquals(1, nil.get(0).entries.get("manager").live);
+
+ Messages.NodeInformationProto nodeInformationProto =
+ RestTypeMarshalling.marshall(ni);
+ Messages.NodeEntryInformationProto entryProto = nodeInformationProto
+ .getEntries(0);
+ assertNotNull(entryProto);
+ assertEquals(1, entryProto.getPriority());
+ NodeInformation unmarshalled =
+ RestTypeMarshalling.unmarshall(nodeInformationProto);
+ assertEquals(unmarshalled.hostname, ni.hostname);
+ assertTrue(unmarshalled.entries.keySet().containsAll(ni.entries.keySet()));
+
+ }
+
+ @Test
+ public void testBuildRolenames() throws Throwable {
+
+ }
+ public List<NodeInstance> assertNoAvailableNodes(int role) {
+ String label = "";
+ return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label));
+ }
+
+ List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) {
+ if (list.size() != size) {
+ for (NodeInstance ni : list) {
+ LOG.error(ni.toFullString());
+ }
+ }
+ assertEquals(size, list.size());
+ return list;
+ }
+
+ NodeMap createNodeMap(List<NodeReport> nodeReports)
+ throws BadConfigException {
+ NodeMap newNodeMap = new NodeMap(1);
+ update(newNodeMap, nodeReports);
+ return newNodeMap;
+ }
+
+ protected boolean update(NodeMap nm, List<NodeReport> nodeReports) {
+ return nm.buildOrUpdate(nodeReports);
+ }
+
+ NodeMap createNodeMap(List<String> hosts, NodeState state,
+ String label) throws BadConfigException {
+ return createNodeMap(MockFactory.INSTANCE.createNodeReports(hosts, state,
+ label));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
new file mode 100644
index 0000000..36a480e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockContainer;
+import org.apache.slider.server.appmaster.model.mock.MockNodeId;
+import org.apache.slider.server.appmaster.state.ContainerOutcome;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.NodeMap;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test container events at the role history level -one below
+ * the App State.
+ */
+public class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryContainerEvents.class);
+
+ @Override
+ public String getTestName() {
+ return "TestRoleHistoryContainerEvents";
+ }
+
+ private NodeInstance age1Active4;
+ private NodeInstance age2Active2;
+ private NodeInstance age3Active0;
+ private NodeInstance age4Active1;
+ private NodeInstance age2Active0;
+
+ private RoleHistory roleHistory;
+
+ private Resource resource;
+
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ return roleHistory.requestContainerForRole(roleStatus).getIssuedRequest();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ age1Active4 = nodeInstance(1, 4, 0, 0);
+ age2Active2 = nodeInstance(2, 2, 0, 1);
+ age3Active0 = nodeInstance(3, 0, 0, 0);
+ age4Active1 = nodeInstance(4, 1, 0, 0);
+ age2Active0 = nodeInstance(2, 0, 0, 0);
+
+ roleHistory = appState.getRoleHistory();
+ roleHistory.insert(Arrays.asList(age2Active2, age2Active0,
+ age4Active1, age1Active4, age3Active0));
+ roleHistory.buildRecentNodeLists();
+ resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
+ ResourceKeys.DEF_YARN_MEMORY);
+ }
+
+ @Test
+ public void testFindAndCreate() throws Throwable {
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ List<String> requestNodes = request.getNodes();
+ assertNotNull(requestNodes);
+ assertEquals(1, requestNodes.size());
+ String hostname = requestNodes.get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ //build a container
+ MockContainer container = factory.newContainer();
+ container.setNodeId(new MockNodeId(hostname, 0));
+ container.setPriority(request.getPriority());
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertEquals(1, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri);
+ //later, declare that it started
+ roleHistory.onContainerStarted(container);
+ assertEquals(0, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ assertEquals(1, roleEntry.getActive());
+ assertEquals(1, roleEntry.getLive());
+ }
+
+ @Test
+ public void testCreateAndRelease() throws Throwable {
+ RoleStatus roleStatus = getRole1Status();
+
+ //verify it is empty
+ assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty());
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ assertNull(request.getNodes());
+
+ //pick an idle host
+ String hostname = age3Active0.hostname;
+
+ //build a container
+ MockContainer container = factory.newContainer(new MockNodeId(hostname,
+ 0), request.getPriority());
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertEquals(1, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri);
+ //later, declare that it started
+ roleHistory.onContainerStarted(container);
+ assertEquals(0, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ assertEquals(1, roleEntry.getActive());
+ assertEquals(1, roleEntry.getLive());
+
+ // now pick that instance to destroy
+ List<NodeInstance> activeNodes = roleHistory.listActiveNodes(roleStatus
+ .getKey());
+
+
+ assertEquals(1, activeNodes.size());
+ NodeInstance target = activeNodes.get(0);
+ assertEquals(target, allocated);
+ roleHistory.onContainerReleaseSubmitted(container);
+ assertEquals(1, roleEntry.getReleasing());
+ assertEquals(1, roleEntry.getLive());
+ assertEquals(0, roleEntry.getActive());
+
+ // release completed
+ roleHistory.onReleaseCompleted(container);
+ assertEquals(0, roleEntry.getReleasing());
+ assertEquals(0, roleEntry.getLive());
+ assertEquals(0, roleEntry.getActive());
+
+ // verify it is empty
+ assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty());
+
+ // ask for a container and expect to get the recently released one
+ AMRMClient.ContainerRequest request2 =
+ requestContainer(roleStatus);
+
+ List<String> nodes2 = request2.getNodes();
+ assertNotNull(nodes2);
+ String hostname2 = nodes2.get(0);
+
+ //pick an idle host
+ assertEquals(hostname2, age3Active0.hostname);
+ }
+
+
+ @Test
+ public void testStartWithoutWarning() throws Throwable {
+ //pick an idle host
+ String hostname = age3Active0.hostname;
+ //build a container
+ MockContainer container = factory.newContainer(
+ new MockNodeId(hostname, 0),
+ ContainerPriority.createPriority(getRole0Status().getKey(), false));
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(getRole0Status().getKey());
+
+ //tell RH that it started
+ roleHistory.onContainerStarted(container);
+ assertEquals(0, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ assertEquals(1, roleEntry.getActive());
+ assertEquals(1, roleEntry.getLive());
+ }
+
+ @Test
+ public void testStartFailed() throws Throwable {
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ LOG.info("req {}", request);
+ LOG.info("{}", request.getNodes());
+ String hostname = request.getNodes().get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ //build a container
+ MockContainer container = factory.newContainer(new MockNodeId(hostname,
+ 0), request.getPriority());
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertEquals(1, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri);
+ //later, declare that it failed on startup
+ assertFalse(roleHistory.onNodeManagerContainerStartFailed(container));
+ assertEquals(0, roleEntry.getStarting());
+ assertEquals(1, roleEntry.getStartFailed());
+ assertEquals(1, roleEntry.getFailed());
+ assertTrue(roleEntry.isAvailable());
+ assertEquals(0, roleEntry.getActive());
+ assertEquals(0, roleEntry.getLive());
+ }
+
+ @Test
+ public void testStartFailedWithoutWarning() throws Throwable {
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ String hostname = request.getNodes().get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ //build a container
+ MockContainer container = factory.newContainer();
+ container.setNodeId(new MockNodeId(hostname, 0));
+ container.setPriority(request.getPriority());
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+
+ assertFalse(roleHistory.onNodeManagerContainerStartFailed(container));
+ assertEquals(0, roleEntry.getStarting());
+ assertEquals(1, roleEntry.getStartFailed());
+ assertEquals(1, roleEntry.getFailed());
+ assertTrue(roleEntry.isAvailable());
+ assertEquals(0, roleEntry.getActive());
+ assertEquals(0, roleEntry.getLive());
+ }
+
+ @Test
+ public void testContainerFailed() throws Throwable {
+ describe("fail a container without declaring it as starting");
+
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ String hostname = request.getNodes().get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ //build a container
+ MockContainer container = factory.newContainer();
+ container.setNodeId(new MockNodeId(hostname, 0));
+ container.setPriority(request.getPriority());
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertEquals(1, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri);
+ roleHistory.onContainerStarted(container);
+
+ //later, declare that it failed
+ roleHistory.onFailedContainer(
+ container,
+ false,
+ ContainerOutcome.Failed);
+ assertEquals(0, roleEntry.getStarting());
+ assertTrue(roleEntry.isAvailable());
+ assertEquals(0, roleEntry.getActive());
+ assertEquals(0, roleEntry.getLive());
+ }
+
+ @Test
+ public void testContainerFailedWithoutWarning() throws Throwable {
+ describe("fail a container without declaring it as starting");
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ String hostname = request.getNodes().get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ //build a container
+ MockContainer container = factory.newContainer();
+ container.setNodeId(new MockNodeId(hostname, 0));
+ container.setPriority(request.getPriority());
+
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertTrue(roleEntry.isAvailable());
+ roleHistory.onFailedContainer(
+ container,
+ false,
+ ContainerOutcome.Failed);
+ assertEquals(0, roleEntry.getStarting());
+ assertEquals(1, roleEntry.getFailed());
+ assertTrue(roleEntry.isAvailable());
+ assertEquals(0, roleEntry.getActive());
+ assertEquals(0, roleEntry.getLive());
+ }
+
+ @Test
+ public void testAllocationListPrep() throws Throwable {
+ describe("test prepareAllocationList");
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ requestContainer(roleStatus);
+
+ String hostname = request.getNodes().get(0);
+ assertEquals(hostname, age3Active0.hostname);
+
+ MockContainer container1 = factory.newContainer();
+ container1.setNodeId(new MockNodeId(hostname, 0));
+ container1.setPriority(Priority.newInstance(getRole0Status().getKey()));
+
+ MockContainer container2 = factory.newContainer();
+ container2.setNodeId(new MockNodeId(hostname, 0));
+ container2.setPriority(Priority.newInstance(getRole1Status().getKey()));
+
+ // put containers in List with role == 1 first
+ List<Container> containers = Arrays.asList((Container) container2,
+ (Container) container1);
+ List<Container> sortedContainers = roleHistory.prepareAllocationList(
+ containers);
+
+ // verify that the first container has role == 0 after sorting
+ MockContainer c1 = (MockContainer) sortedContainers.get(0);
+ assertEquals(getRole0Status().getKey(), c1.getPriority().getPriority());
+ MockContainer c2 = (MockContainer) sortedContainers.get(1);
+ assertEquals(getRole1Status().getKey(), c2.getPriority().getPriority());
+ }
+
+ @Test
+ public void testNodeUpdated() throws Throwable {
+ describe("fail a node");
+
+ RoleStatus roleStatus = getRole0Status();
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestContainerForRole(roleStatus).getIssuedRequest();
+
+ String hostname = request.getNodes().get(0);
+ assertEquals(age3Active0.hostname, hostname);
+
+ // build a container
+ MockContainer container = factory.newContainer(new MockNodeId(hostname,
+ 0), request.getPriority());
+
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname);
+ NodeEntry roleEntry = allocated.get(roleStatus.getKey());
+ assertEquals(1, roleEntry.getStarting());
+ assertFalse(roleEntry.isAvailable());
+ RoleInstance ri = new RoleInstance(container);
+ // start it
+ roleHistory.onContainerStartSubmitted(container, ri);
+ roleHistory.onContainerStarted(container);
+
+ int startSize = nodemap.size();
+
+ // now send a list of updated (failed) nodes event
+ List<NodeReport> nodesUpdated = new ArrayList<>();
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance(hostname, 0),
+ NodeState.LOST,
+ null, null, null, null, 1, null, 0);
+ nodesUpdated.add(nodeReport);
+ roleHistory.onNodesUpdated(nodesUpdated);
+
+ nodemap = roleHistory.cloneNodemap();
+ int endSize = nodemap.size();
+ // as even unused nodes are added to the list, we expect the map size to
+ // be >1
+ assertTrue(startSize <= endSize);
+ assertNotNull(nodemap.get(hostname));
+ assertFalse(nodemap.get(hostname).isOnline());
+
+ // add a failure of a node we've never head of
+ String newhost = "newhost";
+ nodesUpdated = Arrays.asList(
+ NodeReport.newInstance(
+ NodeId.newInstance(newhost, 0),
+ NodeState.LOST,
+ null, null, null, null, 1, null, 0)
+ );
+ roleHistory.onNodesUpdated(nodesUpdated);
+
+ NodeMap nodemap2 = roleHistory.cloneNodemap();
+ assertNotNull(nodemap2.get(newhost));
+ assertFalse(nodemap2.get(newhost).isOnline());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
new file mode 100644
index 0000000..2d49e26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.ContainerOutcome;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Testing finding nodes for new instances.
+ *
+ * This stresses the non-AA codepath
+ */
+public class TestRoleHistoryFindNodesForNewInstances extends
+ BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class);
+
+ public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException {
+ }
+
+ @Override
+ public String getTestName() {
+ return "TestFindNodesForNewInstances";
+ }
+
+ private NodeInstance age1Active4;
+ private NodeInstance age2Active2;
+ private NodeInstance age3Active0;
+ private NodeInstance age4Active1;
+ private NodeInstance age2Active0;
+
+ private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+
+ private RoleStatus roleStat;
+ private RoleStatus roleStat2;
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ age1Active4 = nodeInstance(1, 4, 0, 0);
+ age2Active2 = nodeInstance(2, 2, 0, 1);
+ age3Active0 = nodeInstance(3, 0, 0, 0);
+ age4Active1 = nodeInstance(4, 1, 0, 0);
+ age2Active0 = nodeInstance(2, 0, 0, 0);
+
+ roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
+ age1Active4, age3Active0));
+ roleHistory.buildRecentNodeLists();
+
+ roleStat = getRole0Status();
+ roleStat2 = getRole2Status();
+ }
+
+ public List<NodeInstance> findNodes(int count) {
+ return findNodes(count, roleStat);
+ }
+
+ public List<NodeInstance> findNodes(int count, RoleStatus roleStatus) {
+ List <NodeInstance> found = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus);
+ if (f != null) {
+ found.add(f);
+ }
+ }
+ return found;
+ }
+
+ @Test
+ public void testFind1NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
+ LOG.info("found: {}", found);
+ assertTrue(Arrays.asList(age3Active0).contains(found));
+ }
+
+ @Test
+ public void testFind2NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
+ LOG.info("found: {}", found);
+ assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found));
+ NodeInstance found2 = roleHistory.findRecentNodeForNewInstance(roleStat);
+ LOG.info("found: {}", found2);
+ assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found2));
+ assertNotEquals(found, found2);
+ }
+
+ @Test
+ public void testFind3NodeR0ReturnsNull() throws Throwable {
+ assertEquals(2, findNodes(2).size());
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
+ assertNull(found);
+ }
+
+ @Test
+ public void testFindNodesOneEntry() throws Throwable {
+ List<NodeInstance> foundNodes = findNodes(4, roleStat2);
+ assertEquals(0, foundNodes.size());
+ }
+
+ @Test
+ public void testFindNodesIndependent() throws Throwable {
+ assertEquals(2, findNodes(2).size());
+ roleHistory.dump();
+ assertEquals(0, findNodes(3, roleStat2).size());
+ }
+
+ @Test
+ public void testFindNodesFallsBackWhenUsed() throws Throwable {
+ // mark age2 and active 0 as busy, expect a null back
+ age2Active0.get(getRole0Status().getKey()).onStartCompleted();
+ assertNotEquals(0, age2Active0.getActiveRoleInstances(getRole0Status()
+ .getKey()));
+ age3Active0.get(getRole0Status().getKey()).onStartCompleted();
+ assertNotEquals(0, age3Active0.getActiveRoleInstances(getRole0Status()
+ .getKey()));
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
+ if (found != null) {
+ LOG.info(found.toFullString());
+ }
+ assertNull(found);
+ }
+ @Test
+ public void testFindNodesSkipsFailingNode() throws Throwable {
+ // mark age2 and active 0 as busy, expect a null back
+
+ NodeEntry entry0 = age2Active0.get(getRole0Status().getKey());
+ entry0.containerCompleted(
+ false,
+ ContainerOutcome.Failed);
+ assertTrue(entry0.getFailed() > 0);
+ assertTrue(entry0.getFailedRecently() > 0);
+ entry0.containerCompleted(
+ false,
+ ContainerOutcome.Failed);
+ assertFalse(age2Active0.exceedsFailureThreshold(roleStat));
+ // set failure to 1
+ roleStat.getProviderRole().nodeFailureThreshold = 1;
+ // threshold is now exceeded
+ assertTrue(age2Active0.exceedsFailureThreshold(roleStat));
+
+ // get the role & expect age3 to be picked up, even though it is older
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
+ assertEquals(age3Active0, found);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
new file mode 100644
index 0000000..91abaa4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Unit test to verify the comparators sort as expected.
+ */
+public class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
+
+ private NodeInstance age1Active4;
+ private NodeInstance age2Active2;
+ private NodeInstance age3Active0;
+ private NodeInstance age4Active1;
+ private NodeInstance empty = new NodeInstance("empty", MockFactory
+ .ROLE_COUNT);
+ private NodeInstance age6failing;
+ private NodeInstance age1failing;
+
+ private List<NodeInstance> nodes;
+ private List<NodeInstance> nodesPlusEmpty;
+ private List<NodeInstance> allnodes;
+
+ private RoleStatus role0Status;
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ role0Status = getRole0Status();
+
+ age1Active4 = nodeInstance(1001, 4, 0, 0);
+ age2Active2 = nodeInstance(1002, 2, 0, 0);
+ age3Active0 = nodeInstance(1003, 0, 0, 0);
+ age4Active1 = nodeInstance(1004, 1, 0, 0);
+ age6failing = nodeInstance(1006, 0, 0, 0);
+ age1failing = nodeInstance(1001, 0, 0, 0);
+
+ age6failing.get(role0Status.getKey()).setFailedRecently(2);
+ age1failing.get(role0Status.getKey()).setFailedRecently(1);
+
+ nodes = Arrays.asList(age2Active2, age4Active1, age1Active4, age3Active0);
+ nodesPlusEmpty = Arrays.asList(age2Active2, age4Active1, age1Active4,
+ age3Active0, empty);
+ allnodes = Arrays.asList(age6failing, age2Active2, age4Active1,
+ age1Active4, age3Active0, age1failing);
+ }
+
+ @Override
+ public String getTestName() {
+ return "TestNIComparators";
+ }
+
+ @Test
+ public void testPreferred() throws Throwable {
+ Collections.sort(nodes, new NodeInstance.Preferred(role0Status.getKey()));
+ assertListEquals(nodes, Arrays.asList(age4Active1, age3Active0,
+ age2Active2, age1Active4));
+ }
+
+ /**
+ * The preferred sort still includes failures; up to next phase in process
+ * to handle that.
+ * @throws Throwable
+ */
+ @Test
+ public void testPreferredWithFailures() throws Throwable {
+ Collections.sort(allnodes, new NodeInstance.Preferred(role0Status
+ .getKey()));
+ assertEquals(allnodes.get(0), age6failing);
+ assertEquals(allnodes.get(1), age4Active1);
+ }
+
+ @Test
+ public void testPreferredComparatorDowngradesFailures() throws Throwable {
+ NodeInstance.Preferred preferred = new NodeInstance.Preferred(role0Status
+ .getKey());
+ assertEquals(-1, preferred.compare(age6failing, age1failing));
+ assertEquals(1, preferred.compare(age1failing, age6failing));
+ }
+
+ @Test
+ public void testNewerThanNoRole() throws Throwable {
+ Collections.sort(nodesPlusEmpty, new NodeInstance.Preferred(role0Status
+ .getKey()));
+ assertListEquals(nodesPlusEmpty, Arrays.asList(age4Active1, age3Active0,
+ age2Active2, age1Active4, empty));
+ }
+
+ @Test
+ public void testMoreActiveThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.MoreActiveThan(role0Status
+ .getKey()));
+ assertListEquals(nodes, Arrays.asList(age1Active4, age2Active2,
+ age4Active1, age3Active0));
+ }
+
+ @Test
+ public void testMoreActiveThanEmpty() throws Throwable {
+
+ Collections.sort(nodesPlusEmpty, new NodeInstance.MoreActiveThan(
+ role0Status.getKey()));
+ assertListEquals(nodesPlusEmpty, Arrays.asList(age1Active4, age2Active2,
+ age4Active1, age3Active0, empty));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org