You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/06/21 18:33:42 UTC
[20/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
new file mode 100644
index 0000000..7d8f5a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockContainer;
+import org.apache.slider.server.appmaster.model.mock.MockNodeId;
+import org.apache.slider.server.appmaster.model.mock.MockPriority;
+import org.apache.slider.server.appmaster.model.mock.MockResource;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
+import org.apache.slider.server.appmaster.state.ContainerAllocationResults;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.OutstandingRequest;
+import org.apache.slider.server.appmaster.state.OutstandingRequestTracker;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test outstanding request tracker.
+ */
+public class TestRoleHistoryOutstandingRequestTracker extends
+ BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryOutstandingRequestTracker.class);
+
+ public static final String WORKERS_LABEL = "workers";
+ private NodeInstance host1 = new NodeInstance("host1", 3);
+ private NodeInstance host2 = new NodeInstance("host2", 3);
+ private MockResource resource = factory.newResource(48, 1);
+
+ private OutstandingRequestTracker tracker = new OutstandingRequestTracker();
+
+ public static final String WORKER = "worker";
+
+ @Override
+ public Application buildApplication() {
+ Application application = super.buildApplication();
+ Component component = new Component().name("worker").numberOfContainers(0L);
+ component.getConfiguration().setProperty(ResourceKeys.YARN_LABEL_EXPRESSION,
+ WORKERS_LABEL);
+ application.getComponents().add(component);
+ return application;
+ }
+
+ @Test
+ public void testAddRetrieveEntry() throws Throwable {
+ OutstandingRequest request = tracker.newRequest(host1, 0);
+ assertEquals(tracker.lookupPlacedRequest(0, "host1"), request);
+ assertEquals(tracker.removePlacedRequest(request), request);
+ assertNull(tracker.lookupPlacedRequest(0, "host1"));
+ }
+
+ @Test
+ public void testAddCompleteEntry() throws Throwable {
+ OutstandingRequest req1 = tracker.newRequest(host1, 0);
+ req1.buildContainerRequest(resource, getRole0Status(), 0);
+
+ tracker.newRequest(host2, 0).buildContainerRequest(resource,
+ getRole0Status(), 0);
+ tracker.newRequest(host1, 1).buildContainerRequest(resource,
+ getRole0Status(), 0);
+
+ ContainerAllocationResults allocation = tracker.onContainerAllocated(1,
+ "host1", null);
+ assertEquals(allocation.outcome, ContainerAllocationOutcome.Placed);
+ assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
+
+ assertNull(tracker.lookupPlacedRequest(1, "host1"));
+ assertNotNull(tracker.lookupPlacedRequest(0, "host1"));
+ }
+
+ @Test
+ public void testResetOpenRequests() throws Throwable {
+ OutstandingRequest req1 = tracker.newRequest(null, 0);
+ assertFalse(req1.isLocated());
+ tracker.newRequest(host1, 0);
+ List<OutstandingRequest> openRequests = tracker.listOpenRequests();
+ assertEquals(1, openRequests.size());
+ tracker.resetOutstandingRequests(0);
+ assertTrue(tracker.listOpenRequests().isEmpty());
+ assertTrue(tracker.listPlacedRequests().isEmpty());
+ }
+
+ @Test
+ public void testRemoveOpenRequestUnissued() throws Throwable {
+ OutstandingRequest req1 = tracker.newRequest(null, 0);
+ req1.buildContainerRequest(resource, getRole0Status(), 0);
+ assertEquals(1, tracker.listOpenRequests().size());
+ MockContainer c1 = factory.newContainer(null, new MockPriority(0));
+ c1.setResource(resource);
+
+ ContainerAllocationResults allocation =
+ tracker.onContainerAllocated(0, "host1", c1);
+ ContainerAllocationOutcome outcome = allocation.outcome;
+ assertEquals(outcome, ContainerAllocationOutcome.Unallocated);
+ assertTrue(allocation.operations.isEmpty());
+ assertEquals(1, tracker.listOpenRequests().size());
+ }
+
+ @Test
+ public void testIssuedOpenRequest() throws Throwable {
+ OutstandingRequest req1 = tracker.newRequest(null, 0);
+ req1.buildContainerRequest(resource, getRole0Status(), 0);
+ assertEquals(1, tracker.listOpenRequests().size());
+
+ int pri = ContainerPriority.buildPriority(0, false);
+ assertTrue(pri > 0);
+ MockNodeId nodeId = factory.newNodeId("hostname-1");
+ MockContainer c1 = factory.newContainer(nodeId, new MockPriority(pri));
+
+ c1.setResource(resource);
+
+ ContainerRequest issued = req1.getIssuedRequest();
+ assertEquals(issued.getCapability(), resource);
+ assertEquals(issued.getPriority().getPriority(), c1.getPriority()
+ .getPriority());
+ assertTrue(req1.resourceRequirementsMatch(resource));
+
+ ContainerAllocationResults allocation =
+ tracker.onContainerAllocated(0, nodeId.getHost(), c1);
+ assertEquals(0, tracker.listOpenRequests().size());
+ assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
+
+ assertEquals(allocation.outcome, ContainerAllocationOutcome.Open);
+ assertEquals(allocation.origin, req1);
+ }
+
+ @Test
+ public void testResetEntries() throws Throwable {
+ tracker.newRequest(host1, 0);
+ tracker.newRequest(host2, 0);
+ tracker.newRequest(host1, 1);
+ List<NodeInstance> canceled = tracker.resetOutstandingRequests(0);
+ assertEquals(2, canceled.size());
+ assertTrue(canceled.contains(host1));
+ assertTrue(canceled.contains(host2));
+ assertNotNull(tracker.lookupPlacedRequest(1, "host1"));
+ assertNull(tracker.lookupPlacedRequest(0, "host1"));
+ canceled = tracker.resetOutstandingRequests(0);
+ assertEquals(0, canceled.size());
+ assertEquals(1, tracker.resetOutstandingRequests(1).size());
+ }
+
+ @Test
+ public void testEscalation() throws Throwable {
+ // first request: default placement
+ assertEquals(getRole0Status().getPlacementPolicy(), PlacementPolicy
+ .DEFAULT);
+ Resource res0 = newResource(getRole0Status());
+ OutstandingRequest outstanding0 = tracker.newRequest(host1,
+ getRole0Status().getKey());
+ ContainerRequest initialRequest =
+ outstanding0.buildContainerRequest(res0, getRole0Status(), 0);
+ assertNotNull(outstanding0.getIssuedRequest());
+ assertTrue(outstanding0.isLocated());
+ assertFalse(outstanding0.isEscalated());
+ assertFalse(initialRequest.getRelaxLocality());
+ assertEquals(1, tracker.listPlacedRequests().size());
+
+ // second. This one doesn't get launched. This is to verify that the
+ // escalation process skips entries which are in the list but have not
+ // been issued, which can be a race condition between request issuance &
+ // escalation.
+ // (not one observed outside test authoring, but retained for completeness)
+ Resource res2 = newResource(getRole2Status());
+ OutstandingRequest outstanding2 = tracker.newRequest(host1,
+ getRole2Status().getKey());
+
+ // simulate some time escalation of role 1 MUST now be triggered
+ long interval = getRole0Status().getPlacementTimeoutSeconds() * 1000 + 500;
+ long now = interval;
+ final List<AbstractRMOperation> escalations = tracker
+ .escalateOutstandingRequests(now);
+
+ assertTrue(outstanding0.isEscalated());
+ assertFalse(outstanding2.isEscalated());
+
+ // two entries
+ assertEquals(2, escalations.size());
+ AbstractRMOperation e1 = escalations.get(0);
+ assertTrue(e1 instanceof CancelSingleRequest);
+ final CancelSingleRequest cancel = (CancelSingleRequest) e1;
+ assertEquals(initialRequest, cancel.getRequest());
+ AbstractRMOperation e2 = escalations.get(1);
+ assertTrue(e2 instanceof ContainerRequestOperation);
+ ContainerRequestOperation escRequest = (ContainerRequestOperation) e2;
+ assertTrue(escRequest.getRequest().getRelaxLocality());
+
+ // build that second request from an anti-affine entry
+ // these get placed as well
+ now += interval;
+ ContainerRequest containerReq2 =
+ outstanding2.buildContainerRequest(res2, getRole2Status(), now);
+ // escalate a little bit more
+ final List<AbstractRMOperation> escalations2 = tracker
+ .escalateOutstandingRequests(now);
+ // and expect no new entries
+ assertEquals(0, escalations2.size());
+
+ // go past the role2 timeout
+ now += getRole2Status().getPlacementTimeoutSeconds() * 1000 + 500;
+ // escalate a little bit more
+ final List<AbstractRMOperation> escalations3 = tracker
+ .escalateOutstandingRequests(now);
+ // and expect another escalation
+ assertEquals(2, escalations3.size());
+ assertTrue(outstanding2.isEscalated());
+
+ // finally add a strict entry to the mix
+ Resource res3 = newResource(getRole1Status());
+ OutstandingRequest outstanding3 = tracker.newRequest(host1,
+ getRole1Status().getKey());
+
+ final ProviderRole providerRole1 = getRole1Status().getProviderRole();
+ assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
+ now += interval;
+ assertFalse(outstanding3.mayEscalate());
+ final List<AbstractRMOperation> escalations4 = tracker
+ .escalateOutstandingRequests(now);
+ assertTrue(escalations4.isEmpty());
+
+ }
+
+ /**
+ * If the placement does include a label, the initial request must
+ * <i>not</i> include it.
+ * The escalation request will contain the label, while
+ * leaving out the node list.
+ * retains the node list, but sets relaxLocality==true
+ * @throws Throwable
+ */
+ @Test
+ public void testRequestLabelledPlacement() throws Throwable {
+ NodeInstance ni = new NodeInstance("host1", 0);
+ OutstandingRequest req1 = tracker.newRequest(ni, 0);
+ Resource res0 = factory.newResource(48, 1);
+
+ RoleStatus workerRole = lookupRole(WORKER);
+ // initial request
+ ContainerRequest yarnRequest =
+ req1.buildContainerRequest(res0, workerRole, 0);
+ assertEquals(req1.label, WORKERS_LABEL);
+
+ assertNull(yarnRequest.getNodeLabelExpression());
+ assertFalse(yarnRequest.getRelaxLocality());
+ // escalation
+ ContainerRequest yarnRequest2 = req1.escalate();
+ assertNull(yarnRequest2.getNodes());
+ assertTrue(yarnRequest2.getRelaxLocality());
+ assertEquals(yarnRequest2.getNodeLabelExpression(), WORKERS_LABEL);
+ }
+
+ /**
+ * If the placement doesnt include a label, then the escalation request
+ * retains the node list, but sets relaxLocality==true.
+ * @throws Throwable
+ */
+ @Test
+ public void testRequestUnlabelledPlacement() throws Throwable {
+ NodeInstance ni = new NodeInstance("host1", 0);
+ OutstandingRequest req1 = tracker.newRequest(ni, 0);
+ Resource res0 = factory.newResource(48, 1);
+
+ // initial request
+ ContainerRequest yarnRequest = req1.buildContainerRequest(res0,
+ getRole0Status(), 0);
+ assertNotNull(yarnRequest.getNodes());
+ assertTrue(SliderUtils.isUnset(yarnRequest.getNodeLabelExpression()));
+ assertFalse(yarnRequest.getRelaxLocality());
+ ContainerRequest yarnRequest2 = req1.escalate();
+ assertNotNull(yarnRequest2.getNodes());
+ assertTrue(yarnRequest2.getRelaxLocality());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAARequestNoNodes() throws Throwable {
+ tracker.newAARequest(getRole0Status().getKey(), new ArrayList<>(), "");
+ }
+
+ @Test
+ public void testAARequest() throws Throwable {
+ int role0 = getRole0Status().getKey();
+ OutstandingRequest request = tracker.newAARequest(role0, Arrays
+ .asList(host1), "");
+ assertEquals(host1.hostname, request.hostname);
+ assertFalse(request.isLocated());
+ }
+
+ @Test
+ public void testAARequestPair() throws Throwable {
+ int role0 = getRole0Status().getKey();
+ OutstandingRequest request = tracker.newAARequest(role0, Arrays.asList(
+ host1, host2), "");
+ assertEquals(host1.hostname, request.hostname);
+ assertFalse(request.isLocated());
+ ContainerRequest yarnRequest = request.buildContainerRequest(
+ getRole0Status().copyResourceRequirements(new MockResource(0, 0)),
+ getRole0Status(),
+ 0);
+ assertFalse(yarnRequest.getRelaxLocality());
+ assertFalse(request.mayEscalate());
+
+ assertEquals(2, yarnRequest.getNodes().size());
+ }
+
+ @Test
+ public void testBuildResourceRequirements() throws Throwable {
+ // Store original values
+ Application application = appState.getClusterStatus();
+ Component role0 = application.getComponent(getRole0Status().getGroup());
+ String origMem = role0.getResource().getMemory();
+ Integer origVcores = role0.getResource().getCpus();
+
+ // Resource values to be used for this test
+ int testMem = 32768;
+ int testVcores = 2;
+ role0.resource(new org.apache.slider.api.resource.Resource().memory(Integer
+ .toString(testMem)).cpus(testVcores));
+
+ // Test normalization disabled
+ LOG.info("Test normalization: disabled");
+ role0.getConfiguration().setProperty(
+ ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false");
+ MockResource requestedRes = new MockResource(testMem, testVcores);
+ MockResource expectedRes = new MockResource(testMem, testVcores);
+ LOG.info("Resource requested: {}", requestedRes);
+ Resource resFinal = appState.buildResourceRequirements(getRole0Status());
+ LOG.info("Resource actual: {}", resFinal);
+ assertTrue(Resources.equals(expectedRes, resFinal));
+
+ // Test normalization enabled
+ LOG.info("Test normalization: enabled");
+ role0.getConfiguration().setProperty(
+ ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true");
+ expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores);
+ LOG.info("Resource requested: {}", requestedRes);
+ resFinal = appState.buildResourceRequirements(getRole0Status());
+ LOG.info("Resource actual: {}", resFinal);
+ assertTrue(Resources.equals(expectedRes, resFinal));
+
+ // revert resource configuration to original value
+ role0.resource(new org.apache.slider.api.resource.Resource().memory(origMem)
+ .cpus(origVcores));
+ }
+
+ public Resource newResource(RoleStatus r) {
+ return appState.buildResourceRequirements(r);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
new file mode 100644
index 0000000..a936df5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.avro.LoadedRoleHistory;
+import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test fole history reading and writing.
+ */
+public class TestRoleHistoryRW extends BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryRW.class);
+
+ private static long time = System.currentTimeMillis();
+ public static final String HISTORY_V1_6_ROLE =
+ "org/apache/slider/server/avro/history-v01-6-role.json";
+ public static final String HISTORY_V1_3_ROLE =
+ "org/apache/slider/server/avro/history-v01-3-role.json";
+ public static final String HISTORY_V1B_1_ROLE =
+ "org/apache/slider/server/avro/history_v01b_1_role.json";
+
+ private RoleStatus role0Status;
+ private RoleStatus role1Status;
+
+ static final ProviderRole PROVIDER_ROLE3 = new ProviderRole(
+ "role3",
+ 3,
+ PlacementPolicy.STRICT,
+ 3,
+ 3,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
+
+ @Override
+ public String getTestName() {
+ return "TestHistoryRW";
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ role0Status = getRole0Status();
+ role1Status = getRole1Status();
+ }
+
+ @Test
+ public void testWriteReadEmpty() throws Throwable {
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ roleHistory.onStart(fs, historyPath);
+ Path history = roleHistory.saveHistory(time++);
+ assertTrue(fs.getFileStatus(history).isFile());
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ historyWriter.read(fs, history);
+ }
+
+ @Test
+ public void testWriteReadData() throws Throwable {
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertFalse(roleHistory.onStart(fs, historyPath));
+ String addr = "localhost";
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry ne1 = instance.getOrCreate(0);
+ ne1.setLastUsed(0xf00d);
+
+ Path history = roleHistory.saveHistory(time++);
+ assertTrue(fs.getFileStatus(history).isFile());
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+
+
+ LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
+ assertTrue(0 < loadedRoleHistory.size());
+ rh2.rebuild(loadedRoleHistory);
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+ assertNotNull(ni2);
+ NodeEntry ne2 = ni2.get(0);
+ assertNotNull(ne2);
+ assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
+ }
+
+ @Test
+ public void testWriteReadActiveData() throws Throwable {
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ roleHistory.onStart(fs, historyPath);
+ String addr = "localhost";
+ String addr2 = "rack1server5";
+ NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry orig1 = localhost.getOrCreate(role0Status.getKey());
+ orig1.setLastUsed(0x10);
+ NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2);
+ NodeEntry orig2 = rack1server5.getOrCreate(role1Status.getKey());
+ orig2.setLive(3);
+ assertFalse(orig2.isAvailable());
+ NodeEntry orig3 = localhost.getOrCreate(role1Status.getKey());
+ orig3.setLastUsed(0x20);
+ orig3.setLive(1);
+ assertFalse(orig3.isAvailable());
+ orig3.release();
+ assertTrue(orig3.isAvailable());
+ roleHistory.dump();
+
+ long savetime = 0x0001000;
+ Path history = roleHistory.saveHistory(savetime);
+ assertTrue(fs.getFileStatus(history).isFile());
+ describe("Loaded");
+ LOG.info("testWriteReadActiveData in {}", history);
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+ LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
+ assertEquals(3, loadedRoleHistory.size());
+ rh2.rebuild(loadedRoleHistory);
+ rh2.dump();
+
+ assertEquals(2, rh2.getClusterSize());
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+ assertNotNull(ni2);
+ NodeEntry loadedNE = ni2.get(role0Status.getKey());
+ assertEquals(loadedNE.getLastUsed(), orig1.getLastUsed());
+ NodeInstance ni2b = rh2.getExistingNodeInstance(addr2);
+ assertNotNull(ni2b);
+ NodeEntry loadedNE2 = ni2b.get(role1Status.getKey());
+ assertNotNull(loadedNE2);
+ assertEquals(loadedNE2.getLastUsed(), savetime);
+ assertEquals(rh2.getThawedDataTime(), savetime);
+
+ // now start it
+ rh2.buildRecentNodeLists();
+ describe("starting");
+ rh2.dump();
+ List<NodeInstance> available0 = rh2.cloneRecentNodeList(role0Status
+ .getKey());
+ assertEquals(1, available0.size());
+
+ NodeInstance entry = available0.get(0);
+ assertEquals(entry.hostname, "localhost");
+ assertEquals(entry, localhost);
+ List<NodeInstance> available1 = rh2.cloneRecentNodeList(role1Status
+ .getKey());
+ assertEquals(2, available1.size());
+ //and verify that even if last used was set, the save time is picked up
+ assertEquals(entry.get(role1Status.getKey()).getLastUsed(), roleHistory
+ .getSaveTime());
+
+ }
+
+ @Test
+ public void testWriteThaw() throws Throwable {
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertFalse(roleHistory.onStart(fs, historyPath));
+ String addr = "localhost";
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry ne1 = instance.getOrCreate(0);
+ ne1.setLastUsed(0xf00d);
+
+ Path history = roleHistory.saveHistory(time++);
+ long savetime =roleHistory.getSaveTime();
+ assertTrue(fs.getFileStatus(history).isFile());
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+ assertTrue(rh2.onStart(fs, historyPath));
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+ assertNotNull(ni2);
+ NodeEntry ne2 = ni2.get(0);
+ assertNotNull(ne2);
+ assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
+ assertEquals(rh2.getThawedDataTime(), savetime);
+ }
+
+
+ @Test
+ public void testPurgeOlderEntries() throws Throwable {
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ time = 1;
+ Path file1 = touch(historyWriter, time++);
+ Path file2 = touch(historyWriter, time++);
+ Path file3 = touch(historyWriter, time++);
+ Path file4 = touch(historyWriter, time++);
+ Path file5 = touch(historyWriter, time++);
+ Path file6 = touch(historyWriter, time++);
+
+ assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file1));
+ assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file2));
+ assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file2));
+ assertEquals(3, historyWriter.purgeOlderHistoryEntries(fs, file5));
+ assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file6));
+ try {
+ // make an impossible assertion that will fail if the method
+ // actually completes
+ assertEquals(-1, historyWriter.purgeOlderHistoryEntries(fs, file1));
+ } catch (FileNotFoundException ignored) {
+ // expected
+ }
+
+ }
+
+ public Path touch(RoleHistoryWriter historyWriter, long timeMs)
+ throws IOException {
+ Path path = historyWriter.createHistoryFilename(historyPath, timeMs);
+ FSDataOutputStream out = fs.create(path);
+ out.close();
+ return path;
+ }
+
+ @Test
+ public void testSkipEmptyFileOnRead() throws Throwable {
+ describe("verify that empty histories are skipped on read; old histories " +
+ "purged");
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ roleHistory.onStart(fs, historyPath);
+ time = 0;
+ Path oldhistory = roleHistory.saveHistory(time++);
+
+ String addr = "localhost";
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry ne1 = instance.getOrCreate(0);
+ ne1.setLastUsed(0xf00d);
+
+ Path goodhistory = roleHistory.saveHistory(time++);
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path touched = touch(historyWriter, time++);
+
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+ assertTrue(rh2.onStart(fs, historyPath));
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+ assertNotNull(ni2);
+
+ //and assert the older file got purged
+ assertFalse(fs.exists(oldhistory));
+ assertTrue(fs.exists(goodhistory));
+ assertTrue(fs.exists(touched));
+ }
+
+ @Test
+ public void testSkipBrokenFileOnRead() throws Throwable {
+ describe("verify that empty histories are skipped on read; old histories " +
+ "purged");
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ roleHistory.onStart(fs, historyPath);
+ time = 0;
+ Path oldhistory = roleHistory.saveHistory(time++);
+
+ String addr = "localhost";
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry ne1 = instance.getOrCreate(0);
+ ne1.setLastUsed(0xf00d);
+
+ Path goodhistory = roleHistory.saveHistory(time++);
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path badfile = historyWriter.createHistoryFilename(historyPath, time++);
+ FSDataOutputStream out = fs.create(badfile);
+ out.writeBytes("{broken:true}");
+ out.close();
+
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+ describe("IGNORE STACK TRACE BELOW");
+
+ assertTrue(rh2.onStart(fs, historyPath));
+
+ describe("IGNORE STACK TRACE ABOVE");
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+ assertNotNull(ni2);
+
+ //and assert the older file got purged
+ assertFalse(fs.exists(oldhistory));
+ assertTrue(fs.exists(goodhistory));
+ assertTrue(fs.exists(badfile));
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * matches the current state.
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV13Role() throws Throwable {
+ String source = HISTORY_V1_3_ROLE;
+ RoleHistoryWriter writer = new RoleHistoryWriter();
+
+ LoadedRoleHistory loadedRoleHistory = writer.read(source);
+ assertEquals(4, loadedRoleHistory.size());
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here more roles than expected
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV16Role() throws Throwable {
+ String source = HISTORY_V1_6_ROLE;
+ RoleHistoryWriter writer = new RoleHistoryWriter();
+
+ LoadedRoleHistory loadedRoleHistory = writer.read(source);
+ assertEquals(6, loadedRoleHistory.size());
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertEquals(3, roleHistory.rebuild(loadedRoleHistory));
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * is less than the current state.
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadLessRoles() throws Throwable {
+ String source = HISTORY_V1_3_ROLE;
+ RoleHistoryWriter writer = new RoleHistoryWriter();
+
+ LoadedRoleHistory loadedRoleHistory = writer.read(source);
+ assertEquals(4, loadedRoleHistory.size());
+ List<ProviderRole> expandedRoles = new ArrayList(MockFactory.ROLES);
+ expandedRoles.add(PROVIDER_ROLE3);
+ RoleHistory roleHistory = new MockRoleHistory(expandedRoles);
+ assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+ }
+
+ /**
+ * Test that a v1b JSON file can be read. Here more roles than expected
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV1B1Role() throws Throwable {
+ String source = HISTORY_V1B_1_ROLE;
+ RoleHistoryWriter writer = new RoleHistoryWriter();
+
+ LoadedRoleHistory loadedRoleHistory = writer.read(source);
+ assertEquals(1, loadedRoleHistory.size());
+ assertEquals(2, loadedRoleHistory.roleMap.size());
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
new file mode 100644
index 0000000..0bc2282
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.avro.NewerFilesFirst;
+import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Test role history rw ordering.
+ */
+public class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryRWOrdering.class);
+
+ private List<Path> paths = pathlist(
+ Arrays.asList(
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json"
+ )
+ );
+ private Path h0406c = paths.get(0);
+ private Path h5fffa = paths.get(1);
+ private Path h0001a = paths.get(3);
+
+ public TestRoleHistoryRWOrdering() throws URISyntaxException {
+ }
+
+ List<Path> pathlist(List<String> pathnames) throws URISyntaxException {
+ List<Path> pathList = new ArrayList<>();
+ for (String p : pathnames) {
+ pathList.add(new Path(new URI(p)));
+ }
+ return pathList;
+ }
+
+ @Override
+ public String getTestName() {
+ return "TestHistoryRWOrdering";
+ }
+
+ /**
+ * This tests regexp pattern matching. It uses the current time so isn't
+ * repeatable -but it does test a wider range of values in the process
+ * @throws Throwable
+ */
+ @Test
+ public void testPatternRoundTrip() throws Throwable {
+ describe("test pattern matching of names");
+ long value=System.currentTimeMillis();
+ String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,
+ value);
+ String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN;
+ Pattern pattern = Pattern.compile(matchpattern);
+ Matcher matcher = pattern.matcher(name);
+ if (!matcher.find()) {
+ throw new Exception("No match for pattern $matchpattern in $name");
+ }
+ }
+
+ @Test
+ public void testWriteSequenceReadData() throws Throwable {
+ describe("test that if multiple entries are written, the newest is picked" +
+ " up");
+ long time = System.currentTimeMillis();
+
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ assertFalse(roleHistory.onStart(fs, historyPath));
+ String addr = "localhost";
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+ NodeEntry ne1 = instance.getOrCreate(0);
+ ne1.setLastUsed(0xf00d);
+
+ Path history1 = roleHistory.saveHistory(time++);
+ Path history2 = roleHistory.saveHistory(time++);
+ Path history3 = roleHistory.saveHistory(time);
+
+ //inject a later file with a different name
+ sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello," +
+ " world");
+
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+ List<Path> entries = historyWriter.findAllHistoryEntries(
+ fs,
+ historyPath,
+ false);
+ assertEquals(entries.size(), 3);
+ assertEquals(entries.get(0), history3);
+ assertEquals(entries.get(1), history2);
+ assertEquals(entries.get(2), history1);
+ }
+
+ @Test
+ public void testPathStructure() throws Throwable {
+ assertEquals(h5fffa.getName(), "history-5fffa.json");
+ }
+
+ @Test
+ public void testPathnameComparator() throws Throwable {
+
+ NewerFilesFirst newerName = new NewerFilesFirst();
+
+ LOG.info("{} name is {}", h5fffa, h5fffa.getName());
+ LOG.info("{} name is {}", h0406c, h0406c.getName());
+ assertEquals(newerName.compare(h5fffa, h5fffa), 0);
+ assertTrue(newerName.compare(h5fffa, h0406c) < 0);
+ assertTrue(newerName.compare(h5fffa, h0001a) < 0);
+ assertTrue(newerName.compare(h0001a, h5fffa) > 0);
+
+ }
+
+ @Test
+ public void testPathSort() throws Throwable {
+ List<Path> paths2 = new ArrayList<>(paths);
+ RoleHistoryWriter.sortHistoryPaths(paths2);
+ assertListEquals(paths2,
+ Arrays.asList(
+ paths.get(1),
+ paths.get(0),
+ paths.get(3),
+ paths.get(2)
+ ));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
new file mode 100644
index 0000000..7364201
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockContainer;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.OutstandingRequest;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test the RH availability list and request tracking: that hosts
+ * get removed and added.
+ */
+public class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRoleHistoryRequestTracking.class);
+
+ private String roleName = "test";
+
+ private NodeInstance age1Active4;
+ private NodeInstance age2Active2;
+ private NodeInstance age2Active0;
+ private NodeInstance age3Active0;
+ private NodeInstance age4Active1;
+
+ private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ // 1MB, 1 vcore
+ private Resource resource = Resource.newInstance(1, 1);
+
+ private RoleStatus roleStatus;
+
+ public TestRoleHistoryRequestTracking() throws BadConfigException {
+ }
+
+ AMRMClient.ContainerRequest requestContainer(RoleStatus rs) {
+ return roleHistory.requestContainerForRole(rs).getIssuedRequest();
+ }
+
+ @Override
+ public String getTestName() {
+ return "TestRoleHistoryAvailableList";
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ age1Active4 = nodeInstance(1, 4, 0, 0);
+ age2Active2 = nodeInstance(2, 2, 0, 1);
+ age2Active0 = nodeInstance(2, 0, 0, 0);
+ age3Active0 = nodeInstance(3, 0, 0, 0);
+ age4Active1 = nodeInstance(4, 1, 0, 0);
+
+ roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
+ age1Active4, age3Active0));
+ roleHistory.buildRecentNodeLists();
+ roleStatus = getRole0Status();
+ roleStatus.setResourceRequirements(Resource.newInstance(1, 1));
+ }
+
+ @Test
+ public void testAvailableListBuiltForRoles() throws Throwable {
+ List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(
+ roleStatus.getKey());
+ assertListEquals(Arrays.asList(age3Active0, age2Active0), available0);
+ }
+
+ @Test
+ public void testRequestedNodeOffList() throws Throwable {
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+ assertEquals(age3Active0, ni);
+ assertListEquals(Arrays.asList(age2Active0),
+ roleHistory.cloneRecentNodeList(roleStatus.getKey()));
+ roleHistory.requestInstanceOnNode(ni,
+ roleStatus,
+ resource
+ );
+ }
+
+ @Test
+ public void testRequestedNodeOffListWithFailures() throws Throwable {
+ assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
+
+ NodeEntry age3role0 = recordAsFailed(age3Active0, roleStatus.getKey(), 4);
+ assertTrue(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
+ roleStatus.getNodeFailureThreshold()));
+ recordAsFailed(age2Active0, roleStatus.getKey(), 4);
+ assertTrue(age2Active0.isConsideredUnreliable(roleStatus.getKey(),
+ roleStatus.getNodeFailureThreshold()));
+ // expect to get a null node back
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+ assertNull(ni);
+
+ // which is translated to a no-location request
+ AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
+ roleStatus,
+ resource).getIssuedRequest();
+
+ assertNull(req.getNodes());
+
+ LOG.info("resetting failure count");
+ age3role0.resetFailedRecently();
+ roleHistory.dump();
+ assertEquals(0, age3role0.getFailedRecently());
+ assertFalse(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
+ roleStatus.getNodeFailureThreshold()));
+ assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
+ // looking for a node should now find one
+ ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+ assertEquals(ni, age3Active0);
+ req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource)
+ .getIssuedRequest();
+ assertEquals(1, req.getNodes().size());
+ }
+
+ /**
+ * Verify that strict placement policies generate requests for nodes
+ * irrespective of their failed status.
+ * @throws Throwable
+ */
+ @Test
+ public void testStrictPlacementIgnoresFailures() throws Throwable {
+
+ RoleStatus targetRole = getRole1Status();
+ final ProviderRole providerRole1 = targetRole.getProviderRole();
+ assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
+ int key1 = targetRole.getKey();
+ int key0 = getRole0Status().getKey();
+
+ List<NodeInstance> nodes0 = Arrays.asList(age1Active4, age2Active0,
+ age2Active2, age3Active0, age4Active1);
+ recordAllFailed(key0, 4, nodes0);
+ recordAllFailed(key1, 4, nodes0);
+
+ // trigger a list rebuild
+ roleHistory.buildRecentNodeLists();
+ List<NodeInstance> recentRole0 = roleHistory.cloneRecentNodeList(key0);
+ assertTrue(recentRole0.indexOf(age3Active0) < recentRole0
+ .indexOf(age2Active0));
+
+ // the non-strict role has no suitable nodes
+ assertNull(roleHistory.findRecentNodeForNewInstance(getRole0Status()));
+
+
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(targetRole);
+ assertNotNull(ni);
+
+ NodeInstance ni2 = roleHistory.findRecentNodeForNewInstance(targetRole);
+ assertNotNull(ni2);
+ assertNotEquals(ni, ni2);
+ }
+
+ @Test
+ public void testFindAndRequestNode() throws Throwable {
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+
+ assertEquals(age3Active0.hostname, req.getNodes().get(0));
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
+ .getKey());
+ assertListEquals(Arrays.asList(age2Active0), a2);
+ }
+
+ @Test
+ public void testRequestedNodeIntoReqList() throws Throwable {
+ requestContainer(roleStatus);
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+ assertEquals(1, requests.size());
+ assertEquals(age3Active0.hostname, requests.get(0).hostname);
+ }
+
+ @Test
+ public void testCompletedRequestDropsNode() throws Throwable {
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+ assertEquals(1, requests.size());
+ String hostname = requests.get(0).hostname;
+ assertEquals(age3Active0.hostname, hostname);
+ assertEquals(hostname, req.getNodes().get(0));
+ MockContainer container = factory.newContainer(req, hostname);
+ assertOnContainerAllocated(container, 2, 1);
+ assertNoOutstandingPlacedRequests();
+ }
+
+ public void assertOnContainerAllocated(Container c1, int p1, int p2) {
+ assertNotEquals(ContainerAllocationOutcome.Open, roleHistory
+ .onContainerAllocated(c1, p1, p2).outcome);
+ }
+
+ public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
+ assertEquals(ContainerAllocationOutcome.Open, roleHistory
+ .onContainerAllocated(c1, p1, p2).outcome);
+ }
+
+ void assertNoOutstandingPlacedRequests() {
+ assertTrue(roleHistory.listPlacedRequests().isEmpty());
+ }
+
+ public void assertOutstandingPlacedRequests(int i) {
+ assertEquals(i, roleHistory.listPlacedRequests().size());
+ }
+
+ @Test
+ public void testTwoRequests() throws Throwable {
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+ assertEquals(2, requests.size());
+ MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+ assertOnContainerAllocated(container, 2, 1);
+ assertOutstandingPlacedRequests(1);
+ container = factory.newContainer(req2, req2.getNodes().get(0));
+ assertOnContainerAllocated(container, 2, 2);
+ assertNoOutstandingPlacedRequests();
+ }
+
+ @Test
+ public void testThreeRequestsOneUnsatisified() throws Throwable {
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+ assertEquals(2, requests.size());
+ MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+ assertOnContainerAllocated(container, 2, 1);
+ assertOutstandingPlacedRequests(1);
+
+ container = factory.newContainer(req3, "three");
+ assertOnContainerAllocationOpen(container, 3, 2);
+ assertOutstandingPlacedRequests(1);
+
+ // the final allocation will trigger a cleanup
+ container = factory.newContainer(req2, "four");
+ // no node dropped
+ assertEquals(ContainerAllocationOutcome.Unallocated,
+ roleHistory.onContainerAllocated(container, 3, 3).outcome);
+ // yet the list is now empty
+ assertNoOutstandingPlacedRequests();
+ roleHistory.listOpenRequests().isEmpty();
+
+ // and the remainder goes onto the available list
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
+ .getKey());
+ assertListEquals(Arrays.asList(age2Active0), a2);
+ }
+
+ @Test
+ public void testThreeRequests() throws Throwable {
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
+ assertOutstandingPlacedRequests(2);
+ assertNull(req3.getNodes());
+ MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+ assertOnContainerAllocated(container, 3, 1);
+ assertOutstandingPlacedRequests(1);
+ container = factory.newContainer(req2, req2.getNodes().get(0));
+ assertOnContainerAllocated(container, 3, 2);
+ assertNoOutstandingPlacedRequests();
+ container = factory.newContainer(req3, "three");
+ assertOnContainerAllocationOpen(container, 3, 3);
+ assertNoOutstandingPlacedRequests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
new file mode 100644
index 0000000..33e7930
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAM;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Test updating blacklist.
+ */
+public class TestRoleHistoryUpdateBlacklist extends BaseMockAppStateTest {
+ private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+ private Collection<RoleStatus> roleStatuses;
+ private RoleStatus roleStatus;
+ private NodeInstance ni;
+
+ public TestRoleHistoryUpdateBlacklist() throws BadConfigException {
+ }
+
+ @Override
+ public String getTestName() {
+ return "TestUpdateBlacklist";
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ ni = nodeInstance(1, 0, 0, 0);
+ roleHistory.insert(Arrays.asList(ni));
+ roleHistory.buildRecentNodeLists();
+ appState.setRoleHistory(roleHistory);
+ roleStatus = getRole0Status();
+ roleStatuses = Arrays.asList(roleStatus);
+ }
+
+ @Test
+ public void testUpdateBlacklist() {
+ assertFalse(ni.isBlacklisted());
+
+ // at threshold, blacklist is unmodified
+ recordAsFailed(ni, roleStatus.getKey(), MockFactory.NODE_FAILURE_THRESHOLD);
+ UpdateBlacklistOperation op = roleHistory.updateBlacklist(roleStatuses);
+ assertNull(op);
+ assertFalse(ni.isBlacklisted());
+
+ // threshold is reached, node goes on blacklist
+ recordAsFailed(ni, roleStatus.getKey(), 1);
+ op = roleHistory.updateBlacklist(roleStatuses);
+ assertNotNull(op);
+ assertTrue(ni.isBlacklisted());
+
+ // blacklist remains unmodified
+ op = roleHistory.updateBlacklist(roleStatuses);
+ assertNull(op);
+ assertTrue(ni.isBlacklisted());
+
+ // failure threshold reset, node goes off blacklist
+ ni.resetFailedRecently();
+ op = roleHistory.updateBlacklist(roleStatuses);
+ assertNotNull(op);
+ assertFalse(ni.isBlacklisted());
+ }
+
+ @Test
+ public void testBlacklistOperations()
+ throws Exception {
+ recordAsFailed(ni, roleStatus.getKey(), MockFactory
+ .NODE_FAILURE_THRESHOLD + 1);
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ assertListLength(ops, 1);
+ AbstractRMOperation op = ops.get(0);
+ assertTrue(op instanceof UpdateBlacklistOperation);
+ assertTrue(ni.isBlacklisted());
+
+ MockRMOperationHandler handler = new MockRMOperationHandler();
+ assertEquals(0, handler.getBlacklisted());
+ handler.execute(ops);
+ assertEquals(1, handler.getBlacklisted());
+
+ ResetFailureWindow resetter = new ResetFailureWindow(handler);
+ resetter.execute(new MockAM(), null, appState);
+ assertEquals(0, handler.getBlacklisted());
+ assertFalse(ni.isBlacklisted());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
new file mode 100644
index 0000000..419f2fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.common.tools.SliderUtils;
+
+/**
+ * Provides allocation services to a cluster -both random and placed.
+ *
+ * Important: container allocations need an app attempt ID put into the
+ * container ID
+ */
+public class Allocator {
+
+ private final MockYarnCluster cluster;
+ /**
+ * Rolling index into the cluster used for the next "random" assignment.
+ */
+ private int rollingIndex = 0;
+
+ Allocator(MockYarnCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ /**
+ * Allocate a node using the list of nodes in the container as the
+ * hints.
+ * @param request request
+ * @return the allocated container -or null for none
+ */
+ MockContainer allocate(AMRMClient.ContainerRequest request) {
+ MockYarnCluster.MockYarnClusterNode node = null;
+ MockYarnCluster.MockYarnClusterContainer allocated = null;
+ if (SliderUtils.isNotEmpty(request.getNodes())) {
+ for (String host : request.getNodes()) {
+ node = cluster.lookup(host);
+ allocated = node.allocate();
+ if (allocated != null) {
+ break;
+ }
+ }
+ }
+
+ if (allocated != null) {
+ return createContainerRecord(request, allocated, node);
+ } else {
+ if (request.getRelaxLocality() || request.getNodes().isEmpty()) {
+ // fallback to anywhere
+ return allocateRandom(request);
+ } else {
+ //no match and locality can't be requested
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Allocate a node without any positioning -use whatever policy this allocator
+ * chooses.
+ * @param request request
+ * @return the allocated container -or null for none
+ */
+ MockContainer allocateRandom(AMRMClient.ContainerRequest request) {
+ int start = rollingIndex;
+ MockYarnCluster.MockYarnClusterNode node = cluster.nodeAt(rollingIndex);
+ MockYarnCluster.MockYarnClusterContainer allocated = node.allocate();
+ // if there is no space, try again -but stop when all the nodes
+ // have failed
+ while (allocated == null && start != nextIndex()) {
+ node = cluster.nodeAt(rollingIndex);
+ allocated = node.allocate();
+ }
+
+ //here the allocation is set, so create the response
+ return createContainerRecord(request, allocated, node);
+ }
+
+ /**
+ * Create a container record -if one was allocated.
+ * @param allocated allocation -may be null
+ * @param node node with the container
+ * @return a container record, or null if there was no allocation
+ */
+ public MockContainer createContainerRecord(
+ AMRMClient.ContainerRequest request,
+ MockYarnCluster.MockYarnClusterContainer allocated,
+ MockYarnCluster.MockYarnClusterNode node) {
+ if (allocated == null) {
+ // no space
+ return null;
+ }
+ MockContainer container = new MockContainer();
+ container.setId(new MockContainerId(allocated.getCid()));
+ container.setNodeId(node.getNodeId());
+ container.setNodeHttpAddress(node.httpAddress());
+ container.setPriority(request.getPriority());
+ container.setResource(request.getCapability());
+ return container;
+ }
+
+ public int nextIndex() {
+ rollingIndex = (rollingIndex + 1) % cluster.getClusterSize();
+ return rollingIndex;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
new file mode 100644
index 0000000..eca8401
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.ContainerOutcome;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.NodeMap;
+import org.apache.slider.server.appmaster.state.ProviderAppState;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.utils.SliderTestBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+/**
+ * Base for app state tests.
+ */
+public abstract class BaseMockAppStateTest extends SliderTestBase implements
+ MockRoles {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseMockAppStateTest.class);
+ protected static final List<ContainerId> EMPTY_ID_LIST = Collections
+ .emptyList();
+
+ protected final MockFactory factory = MockFactory.INSTANCE;
+ protected MockAppState appState;
+ protected MockYarnEngine engine;
+ protected FileSystem fs;
+ protected SliderFileSystem sliderFileSystem;
+ protected File historyWorkDir;
+ protected Path historyPath;
+ protected MockApplicationId applicationId;
+ protected MockApplicationAttemptId applicationAttemptId;
+ protected StateAccessForProviders stateAccess;
+
+ /**
+ * Override point: called in setup() to create the YARN engine; can
+ * be changed for different sizes and options.
+ * @return
+ */
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8, 8);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ fs = FileSystem.get(new URI("file:///"), conf);
+ sliderFileSystem = new SliderFileSystem(fs, conf);
+ engine = createYarnEngine();
+ initApp();
+ }
+
+ /**
+ * Initialize the application.
+ * This uses the binding information supplied by {@link #buildBindingInfo()}.
+ */
+ protected void initApp()
+ throws IOException, BadConfigException, BadClusterStateException {
+ String historyDirName = getTestName();
+ applicationId = new MockApplicationId(1, 0);
+ applicationAttemptId = new MockApplicationAttemptId(applicationId, 1);
+
+ historyWorkDir = new File("target/history", historyDirName);
+ historyPath = new Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new MockAppState(buildBindingInfo());
+ stateAccess = new ProviderAppState(getTestName(), appState);
+ }
+
+ /**
+ * Build the binding info from the default constructor values,
+ * the roles from {@link #factory}, and an instance definition.
+ * from {@link #buildApplication()} ()}
+ * @return
+ */
+ protected AppStateBindingInfo buildBindingInfo() {
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.application = buildApplication();
+ //binding.roles = new ArrayList<>(factory.ROLES);
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ binding.nodeReports = engine.getNodeReports();
+ return binding;
+ }
+
+ /**
+ * Override point, define the instance definition.
+ * @return the instance definition
+ */
+ public Application buildApplication() {
+ return factory.newApplication(0, 0, 0).name(getTestName());
+ }
+
+ /**
+ * Get the test name ... defaults to method name
+ * @return the method name
+ */
+ public String getTestName() {
+ return methodName.getMethodName();
+ }
+
+ public RoleStatus getRole0Status() {
+ return lookupRole(ROLE0);
+ }
+
+ public RoleStatus lookupRole(String role) {
+ return appState.lookupRoleStatus(role);
+ }
+
+ public RoleStatus getRole1Status() {
+ return lookupRole(ROLE1);
+ }
+
+ public RoleStatus getRole2Status() {
+ return lookupRole(ROLE2);
+ }
+
+ /**
+ * Build a role instance from a container assignment.
+ * @param assigned
+ * @return the instance
+ */
+ public RoleInstance roleInstance(ContainerAssignment assigned) {
+ Container target = assigned.container;
+ RoleInstance ri = new RoleInstance(target);
+ ri.roleId = assigned.role.getPriority();
+ ri.role = assigned.role.getName();
+ return ri;
+ }
+
+ public NodeInstance nodeInstance(long age, int live0, int live1, int live2) {
+ NodeInstance ni = new NodeInstance(String.format("age%d-[%d,%d,%d]", age,
+ live0, live1, live2), MockFactory.ROLE_COUNT);
+ ni.getOrCreate(getRole0Status().getKey()).setLastUsed(age);
+ ni.getOrCreate(getRole0Status().getKey()).setLive(live0);
+ if (live1 > 0) {
+ ni.getOrCreate(getRole1Status().getKey()).setLive(live1);
+ }
+ if (live2 > 0) {
+ ni.getOrCreate(getRole2Status().getKey()).setLive(live2);
+ }
+ return ni;
+ }
+
+ /**
+ * Create a container status event.
+ * @param c container
+ * @return a status
+ */
+ ContainerStatus containerStatus(Container c) {
+ return containerStatus(c.getId());
+ }
+
+ /**
+ * Create a container status instance for the given ID, declaring
+ * that it was shut down by the application itself.
+ * @param cid container Id
+ * @return the instance
+ */
+ public ContainerStatus containerStatus(ContainerId cid) {
+ ContainerStatus status = containerStatus(cid,
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN);
+ return status;
+ }
+
+ public ContainerStatus containerStatus(ContainerId cid, int exitCode) {
+ ContainerStatus status = ContainerStatus.newInstance(
+ cid,
+ ContainerState.COMPLETE,
+ "",
+ exitCode);
+ return status;
+ }
+
+ /**
+ * Create nodes and bring them to the started state.
+ * @return a list of roles
+ */
+ protected List<RoleInstance> createAndStartNodes()
+ throws TriggerClusterTeardownException, SliderInternalStateException {
+ return createStartAndStopNodes(new ArrayList<>());
+ }
+
+ /**
+ * Create, Start and stop nodes.
+ * @param completionResults List filled in with the status on all completed
+ * nodes
+ * @return the nodes
+ */
+ public List<RoleInstance> createStartAndStopNodes(
+ List<AppState.NodeCompletionResult> completionResults)
+ throws TriggerClusterTeardownException, SliderInternalStateException {
+ List<ContainerId> released = new ArrayList<>();
+ List<RoleInstance> instances = createAndSubmitNodes(released);
+ processSubmissionOperations(instances, completionResults, released);
+ return instances;
+ }
+
+ /**
+ * Process the start/stop operations.
+ * @param instances
+ * @param completionResults
+ * @param released
+ */
+ public void processSubmissionOperations(
+ List<RoleInstance> instances,
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> released) {
+ for (RoleInstance instance : instances) {
+ LOG.debug("Started {} on {}", instance.role, instance.id);
+ assertNotNull(appState.onNodeManagerContainerStarted(instance
+ .getContainerId()));
+ }
+ releaseContainers(completionResults,
+ released,
+ ContainerState.COMPLETE,
+ "released",
+ 0
+ );
+ }
+
+ /**
+ * Release a list of containers, updating the completion results.
+ * @param completionResults
+ * @param containerIds
+ * @param containerState
+ * @param exitText
+ * @param containerExitCode
+ * @return
+ */
+ public void releaseContainers(
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> containerIds,
+ ContainerState containerState,
+ String exitText,
+ int containerExitCode) {
+ for (ContainerId id : containerIds) {
+ ContainerStatus status = ContainerStatus.newInstance(id,
+ containerState,
+ exitText,
+ containerExitCode);
+ completionResults.add(appState.onCompletedContainer(status));
+ }
+ }
+
+ /**
+ * Create nodes and submit them.
+ * @return a list of roles
+ */
+ public List<RoleInstance> createAndSubmitNodes()
+ throws TriggerClusterTeardownException, SliderInternalStateException {
+ return createAndSubmitNodes(new ArrayList<>());
+ }
+
+ /**
+ * Create nodes and submit them.
+ * @return a list of roles
+ */
+ public List<RoleInstance> createAndSubmitNodes(List<ContainerId> containerIds)
+ throws TriggerClusterTeardownException, SliderInternalStateException {
+ return createAndSubmitNodes(containerIds, new ArrayList<>());
+ }
+
+ /**
+ * Create nodes and submit them.
+ * @return a list of roles allocated
+ */
+ public List<RoleInstance> createAndSubmitNodes(
+ List<ContainerId> containerIds,
+ List<AbstractRMOperation> operationsOut)
+ throws TriggerClusterTeardownException, SliderInternalStateException {
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ return submitOperations(ops, containerIds, operationsOut);
+ }
+
+ public List<RoleInstance> submitOperations(
+ List<AbstractRMOperation> operationsIn,
+ List<ContainerId> released) {
+ return submitOperations(operationsIn, released, new ArrayList<>());
+ }
+
+ /**
+ * Process the RM operations and send <code>onContainersAllocated</code>
+ * events to the app state.
+ * @param operationsIn list of incoming ops
+ * @param released released containers
+ * @return list of outbound operations
+ */
+ public List<RoleInstance> submitOperations(
+ List<AbstractRMOperation> operationsIn,
+ List<ContainerId> released,
+ List<AbstractRMOperation> operationsOut) {
+ List<Container> allocatedContainers = engine.execute(operationsIn,
+ released);
+ List<ContainerAssignment> assignments = new ArrayList<>();
+ appState.onContainersAllocated(allocatedContainers, assignments,
+ operationsOut);
+
+ List<RoleInstance> roles = new ArrayList<>();
+ for (ContainerAssignment assigned : assignments) {
+ Container container = assigned.container;
+ RoleInstance ri = roleInstance(assigned);
+ //tell the app it arrived
+ LOG.debug("Start submitted {} on ${}", ri.role, container.getId());
+ appState.containerStartSubmitted(container, ri);
+ roles.add(ri);
+ }
+ return roles;
+ }
+
+ /**
+ * Add the AM to the app state.
+ */
+ protected void addAppMastertoAppState() {
+// appState.buildAppMasterNode(
+// new MockContainerId(applicationAttemptId, 999999L),
+// "appmaster",
+// 0,
+// null);
+ }
+
+ /**
+ * Extract the list of container IDs from the list of role instances.
+ * @param instances instance list
+ * @param role role to look up
+ * @return the list of CIDs
+ */
+ public List<ContainerId> extractContainerIds(
+ List<RoleInstance> instances,
+ String role) {
+ List<ContainerId> ids = new ArrayList<>();
+ for (RoleInstance ri : instances) {
+ if (ri.role.equals(role)) {
+ ids.add(ri.getContainerId());
+ }
+ }
+ return ids;
+ }
+
+ /**
+ * Record a node as failing.
+ * @param node
+ * @param id
+ * @param count
+ * @return the entry
+ */
+ public NodeEntry recordAsFailed(NodeInstance node, int id, int count) {
+ NodeEntry entry = node.getOrCreate(id);
+ for (int i = 1; i <= count; i++) {
+ entry.containerCompleted(
+ false,
+ ContainerOutcome.Failed);
+ }
+ return entry;
+ }
+
+ protected void recordAllFailed(int id, int count, List<NodeInstance> nodes) {
+ for (NodeInstance node : nodes) {
+ recordAsFailed(node, id, count);
+ }
+ }
+
+ /**
+ * Get the container request of an indexed entry. Includes some assertions
+ * for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ public AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops,
+ int index) {
+ assertTrue(index < ops.size());
+ AbstractRMOperation op = ops.get(index);
+ assertTrue(op instanceof ContainerRequestOperation);
+ return ((ContainerRequestOperation) op).getRequest();
+ }
+
+ /**
+ * Get the cancel request of an indexed entry. Includes some assertions for
+ * better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ public AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops,
+ int index) {
+ assertTrue(index < ops.size());
+ AbstractRMOperation op = ops.get(index);
+ assertTrue(op instanceof CancelSingleRequest);
+ return ((CancelSingleRequest) op).getRequest();
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for
+ * the size.
+ * @param ops operations list of size 1
+ * @return the request within the first ContainerRequestOperation
+ */
+ public AMRMClient.ContainerRequest getSingleRequest(
+ List<AbstractRMOperation> ops) {
+ assertEquals(1, ops.size());
+ return getRequest(ops, 0);
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for
+ * the size.
+ * @param ops operations list of size 1
+ * @return the request within the first operation
+ */
+ public AMRMClient.ContainerRequest getSingleCancel(
+ List<AbstractRMOperation> ops) {
+ assertEquals(1, ops.size());
+ return getCancel(ops, 0);
+ }
+
+ /**
+ * Get the single release of a list of operations; includes the check for
+ * the size.
+ * @param ops operations list of size 1
+ * @return the request within the first operation
+ */
+ public ContainerReleaseOperation getSingleRelease(
+ List<AbstractRMOperation> ops) {
+ assertEquals(1, ops.size());
+ AbstractRMOperation op = ops.get(0);
+ assertTrue(op instanceof ContainerReleaseOperation);
+ return (ContainerReleaseOperation) op;
+ }
+
+ /**
+ * Get the node information as a large JSON String.
+ * @return
+ */
+ protected String nodeInformationSnapshotAsString()
+ throws UnsupportedEncodingException, JsonProcessingException {
+ return prettyPrintAsJson(stateAccess.getNodeInformationSnapshot());
+ }
+
+ /**
+ * Scan through all containers and assert that the assignment is AA.
+ * @param index role index
+ */
+ protected void assertAllContainersAA(int index) {
+ for (Entry<String, NodeInstance> nodeMapEntry : cloneNodemap().entrySet()) {
+ String name = nodeMapEntry.getKey();
+ NodeInstance ni = nodeMapEntry.getValue();
+ NodeEntry nodeEntry = ni.get(index);
+ assertTrue("too many instances on node " + name, nodeEntry == null ||
+ nodeEntry.isAntiAffinityConstraintHeld());
+ }
+ }
+
+ /**
+ * Get a snapshot of the nodemap of the application state.
+ * @return a cloned nodemap
+ */
+ protected NodeMap cloneNodemap() {
+ return appState.getRoleHistory().cloneNodemap();
+ }
+
+ /**
+ * Issue a nodes updated event.
+ * @param report report to notify
+ * @return response of AM
+ */
+ protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) {
+ return appState.onNodesUpdated(Collections.singletonList(report));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
new file mode 100644
index 0000000..66ae0f9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+/**
+ * Mock AM.
+ */
+public class MockAM extends SliderAppMaster {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
new file mode 100644
index 0000000..2fcf054
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Extended app state that makes more things public.
+ */
+public class MockAppState extends AppState {
+ public static final int RM_MAX_RAM = 4096;
+ public static final int RM_MAX_CORES = 64;
+
+ private long time = -1;
+
+ public MockAppState(AbstractClusterServices recordFactory) {
+ super(recordFactory, new MetricsAndMonitoring());
+ setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES);
+ }
+
+ /**
+ * Instance with a mock record factory.
+ */
+ public MockAppState() {
+ this(new MockClusterServices());
+ }
+
+ public MockAppState(AppStateBindingInfo bindingInfo)
+ throws BadClusterStateException, IOException, BadConfigException {
+ this();
+ buildInstance(bindingInfo);
+ }
+
+ public Map<String, ProviderRole> getRoleMap() {
+ return super.getRoleMap();
+ }
+
+ /**
+ * Current time. if the <code>time</code> field
+ * is set, that value is returned
+ * @return the current time.
+ */
+ protected long now() {
+ if (time > 0) {
+ return time;
+ }
+ return System.currentTimeMillis();
+ }
+
+ public void setTime(long newTime) {
+ this.time = newTime;
+ }
+
+ public void incTime(long inc) {
+ this.time = this.time + inc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
new file mode 100644
index 0000000..b509625
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+class MockApplicationAttemptId extends ApplicationAttemptId {
+
+ private ApplicationId applicationId;
+ private int attemptId;
+
+ public MockApplicationAttemptId() {
+ }
+
+ public MockApplicationAttemptId(ApplicationId applicationId, int attemptId) {
+ this.applicationId = applicationId;
+ this.attemptId = attemptId;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public int getAttemptId() {
+ return attemptId;
+ }
+
+ @Override
+ public void setAttemptId(int attemptId) {
+ this.attemptId = attemptId;
+ }
+
+ @Override
+ protected void build() {
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org