You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/03/26 19:01:56 UTC
[1/4] incubator-slider git commit: switch to <> constructors in
org/apache/slider/core/conf
Repository: incubator-slider
Updated Branches:
refs/heads/develop 5f397d7da -> f5f837cc7
switch to <> constructors in org/apache/slider/core/conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/63e2b807
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/63e2b807
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/63e2b807
Branch: refs/heads/develop
Commit: 63e2b807ea136f9df62fde7c32dc48932258cdad
Parents: 5f397d7
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 26 14:22:25 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 26 14:22:25 2015 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/core/conf/ConfTree.java | 8 ++++----
.../java/org/apache/slider/core/conf/ConfTreeOperations.java | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/63e2b807/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index c1ce0e8..be7c56f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -56,28 +56,28 @@ public final class ConfTree {
/**
* Metadata
*/
- public Map<String, Object> metadata = new HashMap<String, Object >(INITAL_MAP_CAPACITY);
+ public Map<String, Object> metadata = new HashMap<>(INITAL_MAP_CAPACITY);
/**
* Global options
*/
public Map<String, String> global =
- new HashMap<String, String>(INITAL_MAP_CAPACITY);
+ new HashMap<>(INITAL_MAP_CAPACITY);
/**
* Credentials
*/
public Map<String, List<String>> credentials =
- new HashMap<String, List<String>>(INITAL_MAP_CAPACITY);
+ new HashMap<>(INITAL_MAP_CAPACITY);
/**
* Role options,
* role -> option -> value
*/
public Map<String, Map<String, String>> components =
- new HashMap<String, Map<String, String>>(INITAL_MAP_CAPACITY);
+ new HashMap<>(INITAL_MAP_CAPACITY);
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/63e2b807/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index dca58aa..bc116e7 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -148,7 +148,7 @@ public class ConfTreeOperations {
return operations;
}
//create a new instances
- Map<String, String> map = new HashMap<String, String>();
+ Map<String, String> map = new HashMap<>();
confTree.components.put(name, map);
return new MapOperations(name, map);
}
[4/4] incubator-slider git commit: Merge branch
'feature/SLIDER-832_Jenkins_failing_after_SLIDER-799_merge' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-832_Jenkins_failing_after_SLIDER-799_merge' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f5f837cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f5f837cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f5f837cc
Branch: refs/heads/develop
Commit: f5f837cc74d008becac2a663c13753e65e8a32b8
Parents: 63e2b80 14b90b3
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 26 15:12:55 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 26 15:12:55 2015 +0000
----------------------------------------------------------------------
.../server/appmaster/state/RoleHistory.java | 26 ++++----
...tRoleHistoryOutstandingRequestTracker.groovy | 31 ----------
.../TestRoleHistoryRequestTracking.groovy | 64 ++++++++++++++++----
.../model/mock/BaseMockAppStateTest.groovy | 17 +++++-
.../appmaster/model/mock/MockFactory.groovy | 2 +-
5 files changed, 85 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
[3/4] incubator-slider git commit: SLIDER-832 purge two tests that
are invalid now that the skip-unreliable-nodes logic has moved from
OutstandingRequest up to RoleHistory
Posted by st...@apache.org.
SLIDER-832 purge two tests that are invalid now that the skip-unreliable-nodes logic has moved from OutstandingRequest up to RoleHistory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/14b90b3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/14b90b3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/14b90b3d
Branch: refs/heads/develop
Commit: 14b90b3daaa01a352319b3e73df1f648843aa0b7
Parents: fc7f736
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 26 15:12:41 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 26 15:12:41 2015 +0000
----------------------------------------------------------------------
...tRoleHistoryOutstandingRequestTracker.groovy | 31 --------------------
1 file changed, 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b90b3d/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 3d396f8..99e4190 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.model.history
import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -208,36 +207,6 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
}
- @Test
- public void testPlacementSkipsFailures() throws Throwable {
- final def (res0, outstanding0) = newRequest(role0Status)
- def entry = host1.getOrCreate(role0Status.key)
- entry.containerCompleted(false)
- entry.containerCompleted(false)
- entry.containerCompleted(false)
- assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest =
- outstanding0.buildContainerRequest(res0, role0Status, 0, null)
- assert initialRequest.relaxLocality
- assert initialRequest.nodes == null
- }
-
- @Test
- public void testStrictPlacementDoesntSkipFailures() throws Throwable {
- def roleStatus = role1Status
- assert roleStatus.strictPlacement
- final def (res0, outstanding0) = newRequest(roleStatus)
- def entry = host1.getOrCreate(roleStatus.key)
- entry.containerCompleted(false)
- entry.containerCompleted(false)
- entry.containerCompleted(false)
- assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest =
- outstanding0.buildContainerRequest(res0, roleStatus, 0, null)
- assert !initialRequest.relaxLocality
- assert initialRequest.nodes[0] == host1.hostname
- }
-
/**
* Create a new request (always against host1)
* @param r role status
[2/4] incubator-slider git commit: SLIDER-832 scanning for avaialable
nodes to keep failed nodes on list -just skip them. Tests to show this &
strict placement logic
Posted by st...@apache.org.
SLIDER-832 scanning for avaialable nodes to keep failed nodes on list -just skip them. Tests to show this & strict placement logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fc7f7364
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fc7f7364
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fc7f7364
Branch: refs/heads/develop
Commit: fc7f7364e732c86127e28d0853bf5e4d3e4d6bc9
Parents: 63e2b80
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 26 15:11:36 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 26 15:11:36 2015 +0000
----------------------------------------------------------------------
.../server/appmaster/state/RoleHistory.java | 26 ++++----
.../TestRoleHistoryRequestTracking.groovy | 64 ++++++++++++++++----
.../model/mock/BaseMockAppStateTest.groovy | 17 +++++-
.../appmaster/model/mock/MockFactory.groovy | 2 +-
4 files changed, 85 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc7f7364/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 98cf4e4..c2a741c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -488,30 +488,34 @@ public class RoleHistory {
int roleId = role.getKey();
boolean strictPlacement = role.isStrictPlacement();
NodeInstance nodeInstance = null;
- // get the list of possible targets
+ // Get the list of possible targets.
+ // This is a live list: changes here are preserved
List<NodeInstance> targets = getNodesForRoleId(roleId);
if (targets == null) {
- // add an empty list here for ease downstream
- targets = new ArrayList<>(0);
+ // nothing to allocate on
+ return null;
}
+
int cnt = targets.size();
log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
- // spin until there's a candidate
- while (!targets.isEmpty() && nodeInstance == null) {
- NodeInstance head = targets.remove(0);
- if (head.getActiveRoleInstances(roleId) == 0) {
+ for (int i = 0; i < cnt && nodeInstance == null; i++) {
+ NodeInstance candidate = targets.get(i);
+ if (candidate.getActiveRoleInstances(roleId) == 0) {
// no active instances: check failure statistics
- if (strictPlacement || !head.exceedsFailureThreshold(role)) {
- nodeInstance = head;
+ if (strictPlacement || !candidate.exceedsFailureThreshold(role)) {
+ targets.remove(i);
+ // exit criteria for loop is now met
+ nodeInstance = candidate;
} else {
// too many failures for this node
log.info("Recent node failures is higher than threshold {}. Not requesting host {}",
- role.getNodeFailureThreshold(), head.hostname);
+ role.getNodeFailureThreshold(), candidate.hostname);
}
}
}
+
if (nodeInstance == null) {
- log.info("No historical node found for {}", role.getName());
+ log.info("No node found for {}", role.getName());
}
return nodeInstance;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc7f7364/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 82750a3..9847992 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -18,14 +18,17 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
+import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -37,15 +40,16 @@ import org.junit.Test
* Test the RH availability list and request tracking: that hosts
* get removed and added
*/
+@Slf4j
class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
String roleName = "test"
NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
@@ -87,24 +91,34 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeOffListWithFailures() throws Throwable {
+ assert 0 == roleStatus.key
+ assert !roleHistory.cloneAvailableList(0).isEmpty()
+
+ NodeEntry age3role0 = recordAsFailed(age3Active0, 0, 4)
+ assert age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
+ recordAsFailed(age2Active0, 0, 4)
+ assert age2Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
+ // expect to get a null node back
NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
- assert age3Active0 == ni
+ assert !ni
+
+ // which is translated to a no-location request
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
resource,
"")
- assert 1 == req.nodes.size()
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
- assertListEquals([age2Active0], a2)
- age3Active0.get(0).failedRecently = 4
- req = roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource,
- "")
assertNull(req.nodes)
- age3Active0.get(0).failedRecently = 0
+ log.info "resetting failure count"
+ age3role0.resetFailedRecently()
+ roleHistory.dump()
+ assert 0 == age3role0.failedRecently
+ assert !age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
+ assert !roleHistory.cloneAvailableList(0).isEmpty()
+ // looking for a node should now find one
+ ni = roleHistory.findNodeForNewInstance(roleStatus)
+ assert ni == age3Active0
req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
resource,
@@ -113,6 +127,34 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
}
@Test
+ public void testStrictPlacementIgnoresFailures() throws Throwable {
+
+ def targetRole = role1Status
+ final ProviderRole providerRole1 = targetRole.providerRole
+ assert providerRole1.placementPolicy == PlacementPolicy.STRICT
+ int key = targetRole.key
+
+ recordAsFailed(age1Active4, key, 4)
+ recordAsFailed(age2Active0, key, 4)
+ recordAsFailed(age2Active2, key, 4)
+ recordAsFailed(age3Active0, key, 4)
+ recordAsFailed(age4Active1, key, 4)
+
+ // trigger a list rebuild
+ roleHistory.buildAvailableNodeLists();
+
+ assert !roleHistory.cloneAvailableList(key).isEmpty()
+
+
+ NodeInstance ni = roleHistory.findNodeForNewInstance(targetRole)
+ assert ni == age4Active1!= null
+ // next lookup returns next node
+ ni = roleHistory.findNodeForNewInstance(roleStatus)
+ assert ni == age3Active0
+ }
+
+
+ @Test
public void testFindAndRequestNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc7f7364/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index f30fce6..3e5494f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -35,6 +35,7 @@ import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.RoleStatus
@@ -147,7 +148,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
public NodeInstance nodeInstance(long age, int live0, int live1=0, int live2=0) {
- NodeInstance ni = new NodeInstance("age${age}live[${live0},${live1},$live2]",
+ NodeInstance ni = new NodeInstance("age${age}-[${live0},${live1},$live2]",
MockFactory.ROLE_COUNT)
ni.getOrCreate(0).lastUsed = age
ni.getOrCreate(0).live = live0;
@@ -333,4 +334,18 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
return cids
}
+ /**
+ * Record a node as failing
+ * @param node
+ * @param id
+ * @param count
+ * @return the entry
+ */
+ public NodeEntry recordAsFailed(NodeInstance node, int id, int count) {
+ def entry = node.getOrCreate(id)
+ 1.upto(count) {
+ entry.containerCompleted(false)
+ }
+ entry
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc7f7364/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 06bc10c..fca3376 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -184,7 +184,7 @@ class MockFactory implements MockRoles {
def roleMap(int count) {
return [
- (ResourceKeys.COMPONENT_INSTANCES):count.toString(),
+ (ResourceKeys.COMPONENT_INSTANCES): count.toString(),
]
}