You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/23 18:20:25 UTC
[13/50] incubator-slider git commit: SLIDER-967 tests on rolehistory
& nodemap. low-level tests work,
but the app state ones aren't tagging requested nodes as unavailable
SLIDER-967 tests on rolehistory & nodemap. low-level tests work, but the app state ones aren't tagging requested nodes as unavailable
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a7ba72e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a7ba72e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a7ba72e0
Branch: refs/heads/develop
Commit: a7ba72e0ede14b70806b44d61d81bc4357052dbc
Parents: 7899f59
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 12 12:03:11 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 12 12:03:11 2015 +0000
----------------------------------------------------------------------
.../slider/api/types/NodeEntryInformation.java | 18 +++++
.../server/appmaster/state/NodeInstance.java | 16 +++-
.../slider/server/appmaster/state/NodeMap.java | 1 +
.../server/appmaster/state/RoleHistory.java | 2 +-
.../appstate/TestMockAppStateAAPlacement.groovy | 18 ++++-
.../TestMockAppStateContainerFailure.groovy | 2 +-
.../model/history/TestRoleHistoryAA.groovy | 81 +++++++++++++++++---
.../model/mock/BaseMockAppStateTest.groovy | 4 +-
8 files changed, 125 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
index 482b0c7..15b57b0 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
@@ -58,4 +58,22 @@ public class NodeEntryInformation {
/** number of starting instances */
public int starting;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "NodeEntryInformation{");
+ sb.append("priority=").append(priority);
+ sb.append(", live=").append(live);
+ sb.append(", requested=").append(requested);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", starting=").append(starting);
+ sb.append(", failed=").append(failed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", preempted=").append(preempted);
+ sb.append(", lastUsed=").append(lastUsed);
+ sb.append('}');
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index ebd9d5a..8110bff 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -205,7 +205,6 @@ public class NodeInstance {
nodeEntries.add(nodeEntry);
}
-
/**
* run through each entry; gc'ing & removing old ones that don't have
* a recent failure count (we care about those)
@@ -381,5 +380,20 @@ public class NodeInstance {
return activeRight - activeLeft;
}
}
+ /**
+ * A comparator for sorting entries alphabetically
+ */
+ public static class CompareNames implements Comparator<NodeInstance>,
+ Serializable {
+
+ public CompareNames() {
+ }
+
+ @Override
+ public int compare(NodeInstance left, NodeInstance right) {
+ return left.hostname.compareTo(right.hostname);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index aea48b3..23411ca 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -155,6 +155,7 @@ public class NodeMap extends HashMap<String, NodeInstance> {
nodes.add(instance);
}
}
+ Collections.sort(nodes, new NodeInstance.CompareNames());
return nodes;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2ca5367..8a840fc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -512,7 +512,7 @@ public class RoleHistory {
}
/**
- * Get a possibly emtpy list of suggested nodes for a role.
+ * Get a possibly empty list of suggested nodes for a role.
* @param id role ID
* @return list
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index c7f59e3..64c0362 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -30,6 +30,7 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
@@ -105,6 +106,13 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
List<AbstractRMOperation> operations = []
appState.onContainersAllocated([allocated], assignments, operations)
+ def host = allocated.nodeId.host
+ def hostInstance = nodemap.get(host)
+ assert hostInstance.get(aaRole.key).starting == 1
+ assert !hostInstance.canHost(aaRole.key, "")
+ assert !hostInstance.canHost(aaRole.key, null)
+
+
// assignment
assert assignments.size() == 1
@@ -115,6 +123,10 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
// we also expect a new allocation request to have been issued
def req2 = getRequest(operations, 1)
+ assert req2.nodes.size() == engine.cluster.clusterSize - 1
+
+ assert !req2.nodes.contains(host)
+ assert !request.relaxLocality
// verify the pending couner is down
assert 0L == aaRole.pendingAntiAffineRequests
@@ -149,11 +161,13 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert ops2.empty
assert aaRole.pendingAntiAffineRequests == 2
+ assertAllContainersAA()
// next iter
assert 1 == submitOperations(ops, [], ops2).size()
assert 2 == ops2.size()
assert aaRole.pendingAntiAffineRequests == 1
+ assertAllContainersAA()
assert 0 == appState.reviewRequestAndReleaseNodes().size()
// now trigger the next execution cycle
@@ -187,6 +201,8 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
// next iter
submitOperations(ops, [], ops2).size()
assert 1 == ops2.size()
+ assertAllContainersAA()
+
}
/**
@@ -230,7 +246,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
nodemap.each { name, info ->
def nodeEntry = info.entries[index]
assert nodeEntry == null ||
- (nodeEntry.live + nodeEntry.starting + nodeEntry.releasing) <= 1 ,
+ (nodeEntry.live -nodeEntry.releasing + nodeEntry.starting) <= 1 ,
"too many instances on node $name"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 5b24a59..3235827 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -58,7 +58,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest
*/
@Override
MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8000, 4)
+ return new MockYarnEngine(4, 8000)
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
index fdbc3b4..9d0efa2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -58,32 +58,40 @@ class TestRoleHistoryAA extends SliderTestBase {
@Test
public void testFindNodesInFullCluster() throws Throwable {
// all three will surface at first
- assertResultSize(3, nodeMap.findAllNodesForRole(1, ""))
+ verifyResultSize(3, nodeMap.findAllNodesForRole(1, ""))
}
@Test
public void testFindNodesInUnhealthyCluster() throws Throwable {
// all three will surface at first
- nodeMap.get("one").updateNode(new MockNodeReport("one",NodeState.UNHEALTHY))
- assertResultSize(2, nodeMap.findAllNodesForRole(1, ""))
+ markNodeOneUnhealthy()
+ verifyResultSize(2, nodeMap.findAllNodesForRole(1, ""))
+ }
+
+ public boolean markNodeOneUnhealthy() {
+ return setNodeState(nodeMap.get("one"), NodeState.UNHEALTHY)
+ }
+
+ protected boolean setNodeState(NodeInstance node, NodeState state) {
+ node.updateNode(new MockNodeReport(node.hostname, state))
}
@Test
public void testFindNoNodesWrongLabel() throws Throwable {
// all three will surface at first
- assertResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
}
@Test
public void testFindNoNodesRightLabel() throws Throwable {
// all three will surface at first
- assertResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"))
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"))
}
@Test
public void testFindNoNodesNoLabel() throws Throwable {
// all three will surface at first
- assertResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""))
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""))
}
@Test
@@ -92,7 +100,7 @@ class TestRoleHistoryAA extends SliderTestBase {
applyToNodeEntries(nodeMap) {
NodeEntry it -> it.request()
}
- assertResultSize(0, nodeMap.findAllNodesForRole(1, ""))
+ assertNoAvailableNodes(1)
}
@Test
@@ -101,14 +109,67 @@ class TestRoleHistoryAA extends SliderTestBase {
applyToNodeEntries(nodeMap) {
NodeEntry it -> it.request()
}
- assertResultSize(0, nodeMap.findAllNodesForRole(1, ""))
+ assertNoAvailableNodes(1)
+ }
+
+ /**
+ * Tag all nodes as starting, then walk one through a bit
+ * more of its lifecycle
+ */
+ @Test
+ public void testFindNoNodesLifecycle() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.onStarting()
+ }
+ assertNoAvailableNodes(1)
+
+ // walk one of the nodes through the lifecycle
+ def node1 = nodeMap.get("one")
+ assert !node1.canHost(1,"")
+ node1.get(1).onStartCompleted()
+ assert !node1.canHost(1,"")
+ assertNoAvailableNodes()
+ node1.get(1).release()
+ assert node1.canHost(1,"")
+ def list2 = verifyResultSize(1, nodeMap.findAllNodesForRole(1, ""))
+ assert list2[0].hostname == "one"
+
+ // now tag that node as unhealthy and expect it to go away
+ markNodeOneUnhealthy()
+ assertNoAvailableNodes()
+ }
+
+ @Test
+ public void testRolesIndependent() throws Throwable {
+ def node1 = nodeMap.get("one")
+ def role1 = node1.getOrCreate(1)
+ def role2 = node1.getOrCreate(2)
+ nodeMap.values().each {
+ it.updateNode(new MockNodeReport("", NodeState.UNHEALTHY))
+ }
+ assertNoAvailableNodes(1)
+ assertNoAvailableNodes(2)
+ assert setNodeState(node1, NodeState.RUNNING)
+ // tag role 1 as busy
+ role1.onStarting()
+ assertNoAvailableNodes(1)
+
+ verifyResultSize(1, nodeMap.findAllNodesForRole(2, ""))
+ assert node1.canHost(2,"")
+ }
+
+
+ public List<NodeInstance> assertNoAvailableNodes(int role = 1, String label = "") {
+ return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label))
}
- def assertResultSize(int size, List<NodeInstance> list) {
+ List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) {
if (list.size() != size) {
- list.each { log.error(it.toFullString())}
+ list.each { log.error(it.toFullString()) }
}
assert size == list.size()
+ list
}
def applyToNodeEntries(Collection<NodeInstance> list, Closure cl) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7ba72e0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index e1660ee..3d472f1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -69,10 +69,9 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
* @return
*/
public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(64, 1)
+ return new MockYarnEngine(8, 8)
}
-
@Override
void setup() {
super.setup()
@@ -89,7 +88,6 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
*/
void initApp(){
String historyDirName = testName;
- YarnConfiguration conf = SliderUtils.createConfiguration()
applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0)
applicationAttemptId = new MockApplicationAttemptId(
applicationId: applicationId,