You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/03/24 21:01:33 UTC
[01/20] incubator-slider git commit: SLIDER-799 clarify in javadocs
that an RTE is raised if a container with no hostname is looked up. Add a new
method getExistingNodeInstance() to find the node instance hosting a
container, if it exists
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-799-AM-managed-relax [created] b952b6401
SLIDER-799 clarify in javadocs that an RTE is raised if a container with no hostname is looked up. Add a new method getExistingNodeInstance() to find the node instance hosting a container, if it exists
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/55815c22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/55815c22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/55815c22
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 55815c227912602d80230971cbcd2fda3742cd79
Parents: 2037ac4
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 3 15:25:17 2015 -0800
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 3 15:25:17 2015 -0800
----------------------------------------------------------------------
.../server/appmaster/state/RoleHistory.java | 21 ++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55815c22/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index e94457a..5509cac 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -47,13 +47,13 @@ import com.google.common.annotations.VisibleForTesting;
/**
* The Role History.
- *
+ * <p>
* Synchronization policy: all public operations are synchronized.
* Protected methods are in place for testing -no guarantees are made.
- *
+ * <p>
* Inner classes have no synchronization guarantees; they should be manipulated
* in these classes and not externally.
- *
+ * <p>
* Note that as well as some methods marked visible for testing, there
* is the option for the time generator method, {@link #now()} to
* be overridden so that a repeatable time series can be used.
@@ -573,6 +573,7 @@ public class RoleHistory {
* Get the node entry of a container
* @param container container to look up
* @return the entry
+ * @throws RuntimeException if the container has no hostname
*/
public NodeEntry getOrCreateNodeEntry(Container container) {
NodeInstance node = getOrCreateNodeInstance(container);
@@ -583,6 +584,7 @@ public class RoleHistory {
* Get the node instance of a container -always returns something
* @param container container to look up
* @return a (possibly new) node instance
+ * @throws RuntimeException if the container has no hostname
*/
public synchronized NodeInstance getOrCreateNodeInstance(Container container) {
String hostname = RoleHistoryUtils.hostnameOf(container);
@@ -593,12 +595,23 @@ public class RoleHistory {
* Get the node instance of a host if defined
* @param hostname hostname to look up
* @return a node instance or null
+ * @throws RuntimeException if the container has no hostname
*/
public synchronized NodeInstance getExistingNodeInstance(String hostname) {
return nodemap.get(hostname);
}
/**
+ * Get the node instance of a container <i>if there's an entry in the history</i>
+ * @param container container to look up
+ * @return a node instance or null
+ * @throws RuntimeException if the container has no hostname
+ */
+ public synchronized NodeInstance getExistingNodeInstance(Container container) {
+ return nodemap.get(RoleHistoryUtils.hostnameOf(container));
+ }
+
+ /**
* Perform any pre-allocation operations on the list of allocated containers
* based on knowledge of system state.
* Currently this places requested hosts ahead of unrequested ones.
@@ -670,7 +683,7 @@ public class RoleHistory {
/**
* Container start event
- * @param container
+ * @param container container that just started
*/
public void onContainerStarted(Container container) {
NodeEntry nodeEntry = getOrCreateNodeEntry(container);
[13/20] incubator-slider git commit: SLIDER-799 quick code review of
this while looking @ log aggregation setup
Posted by st...@apache.org.
SLIDER-799 quick code review of this while looking @ log aggregation setup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9e671614
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9e671614
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9e671614
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 9e671614f24570ecfedb50a5eaf52e52e74825b4
Parents: 06ac342
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 10:52:21 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 10:52:21 2015 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/core/main/ServiceShutdownHook.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9e671614/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
index 83448ad..de55789 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
@@ -40,7 +40,7 @@ public class ServiceShutdownHook implements Runnable {
private Runnable hook;
public ServiceShutdownHook(Service service) {
- serviceRef = new WeakReference<Service>(service);
+ serviceRef = new WeakReference<>(service);
}
public void register(int priority) {
@@ -74,7 +74,7 @@ public class ServiceShutdownHook implements Runnable {
// Stop the Service
service.stop();
} catch (Throwable t) {
- LOG.info("Error stopping {}: {}", service.getName(), t);
+ LOG.info("Error stopping {}", service.getName(), t);
}
}
}
[07/20] incubator-slider git commit: SLIDER-799 moving towards
escalation;
this code has a sort algorithm for the node target list that compares failure
history before age -about to change this
Posted by st...@apache.org.
SLIDER-799 moving towards escalation; this code has a sort algorithm for the node target list that compares failure history before age -about to change this
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/559ed00a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/559ed00a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/559ed00a
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 559ed00a6b9f3aa8c2ec6c99eb3f85e9807f63ad
Parents: a0d76b7
Author: Steve Loughran <st...@apache.org>
Authored: Mon Mar 16 16:35:46 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Mar 16 16:35:46 2015 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 9 +--
.../MostRecentContainerReleaseSelector.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 62 ++++++++++++---
.../slider/server/appmaster/state/NodeMap.java | 4 +-
.../appmaster/state/OutstandingRequest.java | 83 +++++++++++++++++---
.../state/OutstandingRequestTracker.java | 27 ++++---
.../appmaster/state/ProviderAppState.java | 8 +-
.../server/appmaster/state/RoleHistory.java | 53 ++++++-------
.../server/appmaster/state/RoleStatus.java | 4 +
.../slider/server/avro/NewerFilesFirst.java | 43 ++++++++++
.../slider/server/avro/OlderFilesFirst.java | 43 ++++++++++
.../slider/server/avro/RoleHistoryWriter.java | 62 +++------------
.../TestRoleHistoryContainerEvents.groovy | 4 +-
.../history/TestRoleHistoryNIComparators.groovy | 25 +++++-
...tRoleHistoryOutstandingRequestTracker.groovy | 14 ++--
.../history/TestRoleHistoryRWOrdering.groovy | 3 +-
16 files changed, 305 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 4713ef1..f2c237c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -745,7 +745,7 @@ public class AppState {
*/
private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
- List<ProviderRole> newRoles = new ArrayList<ProviderRole>(0);
+ List<ProviderRole> newRoles = new ArrayList<>(0);
//now update every role's desired count.
//if there are no instance values, that role count goes to zero
@@ -992,7 +992,7 @@ public class AppState {
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
Collection<RoleInstance> values = getLiveNodes().values();
- allRoleInstances = new ArrayList<RoleInstance>(values);
+ allRoleInstances = new ArrayList<>(values);
return allRoleInstances;
}
@@ -1220,7 +1220,6 @@ public class AppState {
AMRMClient.ContainerRequest request;
request = roleHistory.requestNode(role, resource, labelExpression);
incrementRequestCount(role);
-
return request;
}
@@ -1509,7 +1508,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container, true);
+ roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1873,7 +1872,7 @@ public class AppState {
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
- List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
+ List<AbstractRMOperation> operations = new ArrayList<>();
int delta;
String details;
int expected;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
index 841dda3..9d936a1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -40,7 +40,7 @@ public class MostRecentContainerReleaseSelector implements ContainerReleaseSelec
private static class newerThan implements Comparator<RoleInstance>, Serializable {
private final Comparator<Long> innerComparator =
- new Comparators.ComparatorReverser<Long>(new Comparators.LongComparator());
+ new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
public int compare(RoleInstance o1, RoleInstance o2) {
return innerComparator.compare(o1.createTime, o2.createTime);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 231865e..71b74fc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -96,6 +96,22 @@ public class NodeInstance {
}
/**
+ * Query for a node being considered unreliable
+ * @param role role key
+ * @param threshold threshold above which a node is considered unreliable
+ * @return true if the node is considered unreliable
+ */
+ public boolean isConsideredUnreliable(int role, int threshold) {
+ NodeEntry entry = get(role);
+
+ if (entry != null) {
+ return entry.getFailedRecently() > threshold;
+ } else {
+ return false;
+ }
+ }
+
+ /**
* Get the entry for a role -and remove it if present
* @param role the role index
* @return the entry that WAS there
@@ -113,6 +129,7 @@ public class NodeInstance {
nodeEntries.add(nodeEntry);
}
+
/**
* run through each entry; gc'ing & removing old ones
* @param absoluteTime age in millis
@@ -185,28 +202,49 @@ public class NodeInstance {
}
/**
- * A comparator for sorting entries where the role is newer than
- * the other.
- * This sort only compares the lastUsed field, not whether the
- * node is in use or not
+ * A comparator for sorting entries where the node is preferred over another.
+ * If there's no entry for an element then it's failure count is set to -1, age to 0
+ * for the purposes of the comparison
+ * <ol>
+ * <li>Entry exists => end of list as unknown</li>
+ * <li>Lowest failure count</li>
+ * <li>Age</li>
+ * </ol>
+ *
+ * @return +ve int if left is preferred to right; -ve if right over left, 0 for equal
*/
- public static class newerThan implements Comparator<NodeInstance>,
+ public static class Preferred implements Comparator<NodeInstance>,
Serializable {
final int role;
- public newerThan(int role) {
+ public Preferred(int role) {
this.role = role;
}
@Override
public int compare(NodeInstance o1, NodeInstance o2) {
- long age = o1.getOrCreate(role).getLastUsed();
- long age2 = o2.getOrCreate(role).getLastUsed();
+ NodeEntry left = o1.get(role);
+ NodeEntry right = o2.get(role);
+
+ // sort by failure count first
+ int failL = left != null ? left.getFailedRecently() : -1;
+ int failR = right != null ? right.getFailedRecently() : -1;
+
+ if (failL < failR) {
+ return 1;
+ }
+ if (failR > failL) {
+ return -1;
+ }
+
+ // failure counts are equal: compare age
+ long ageL = left != null ? left.getLastUsed() : 0;
+ long ageR = right != null ? right.getLastUsed() : 0;
- if (age > age2) {
+ if (ageL > ageR) {
return -1;
- } else if (age < age2) {
+ } else if (ageL < ageR) {
return 1;
}
// equal
@@ -220,12 +258,12 @@ public class NodeInstance {
* This sort only compares the lastUsed field, not whether the
* node is in use or not
*/
- public static class moreActiveThan implements Comparator<NodeInstance>,
+ public static class MoreActiveThan implements Comparator<NodeInstance>,
Serializable {
final int role;
- public moreActiveThan(int role) {
+ public MoreActiveThan(int role) {
this.role = role;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index fe40086..aa50baa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -73,13 +73,13 @@ public class NodeMap extends HashMap<String, NodeInstance> {
* in that role
*/
public List<NodeInstance> listActiveNodes(int role) {
- List<NodeInstance> nodes = new ArrayList<NodeInstance>();
+ List<NodeInstance> nodes = new ArrayList<>();
for (NodeInstance instance : values()) {
if (instance.getActiveRoleInstances(role) > 0) {
nodes.add(instance);
}
}
- Collections.sort(nodes, new NodeInstance.moreActiveThan(role));
+ Collections.sort(nodes, new NodeInstance.MoreActiveThan(role));
return nodes;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index c6a9cff..f6b83a7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -19,12 +19,17 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
/**
* Tracks an outstanding request. This is used to correlate an allocation response
* with the node and role used in the request.
@@ -50,18 +55,35 @@ public final class OutstandingRequest {
* Node the request is for -may be null
*/
public final NodeInstance node;
+
/**
* hostname -will be null if node==null
*/
public final String hostname;
/**
- * requested time -only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Requested time in millis.
+ * <p>
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ */
+ public AMRMClient.ContainerRequest issuedRequest;
+
+ /**
+ * Requested time in millis.
+ * <p>
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ */
+ public long requestedTimeMillis;
+
+ /**
+ * Time in millis after which escalation should be triggered..
+ * <p>
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
*/
- public long requestedTime;
+ public long escalationTimeoutMillis;
/**
- * Has the placement request been escalated by cancel and re-request
+ * Has the placement request been escalated?
*/
public boolean escalated;
@@ -91,9 +113,14 @@ public final class OutstandingRequest {
this.hostname = hostname;
}
+ /**
+ * Is the request located in the cluster, that is: does it have a node.
+ * @return
+ */
public boolean isLocated() {
return node != null;
}
+
/**
* Build a container request.
* If the request has an address, it is set in the container request
@@ -103,7 +130,7 @@ public final class OutstandingRequest {
* on outstanding requests
* @param resource resource
* @param role role
- * @param time time to record as request time
+ * @param time time in millis to record as request time
* @param labelExpression label to satisfy
* @return the request to raise
*/
@@ -111,7 +138,8 @@ public final class OutstandingRequest {
Resource resource, RoleStatus role, long time, String labelExpression) {
String[] hosts;
boolean relaxLocality;
- requestedTime = time;
+ requestedTimeMillis = time;
+ escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
boolean usePlacementHistory = role.isStrictPlacement();
if (!usePlacementHistory) {
// If strict placement does not mandate using placement then check
@@ -133,23 +161,56 @@ public final class OutstandingRequest {
// tell the node it is in play
node.getOrCreate(roleId);
log.info("Submitting request for container on {}", hosts[0]);
+ escalated = false;
} else {
+ // the placement is implicitly escalated.
+ escalated = true;
hosts = null;
relaxLocality = true;
}
- Priority pri = ContainerPriority.createPriority(roleId,
- !relaxLocality);
- AMRMClient.ContainerRequest request =
- new AMRMClient.ContainerRequest(resource,
+ Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
+ issuedRequest = new AMRMClient.ContainerRequest(resource,
hosts,
null,
pri,
relaxLocality,
labelExpression);
- return request;
+ return issuedRequest;
}
+
+ /**
+ * Build an escalated container request, updating {@link #issuedRequest} with
+ * the new value.
+ * @return the new container request, which has the same resource and label requirements
+ * as the original one, and the same host, but: relaxed placement, and a changed priority
+ * so as to place it into the relaxed list.
+ */
+ public AMRMClient.ContainerRequest buildEscalatedContainerRequest() {
+ escalated = true;
+ Preconditions.checkNotNull(issuedRequest, "issued request");
+ Priority pri = ContainerPriority.createPriority(roleId, true);
+ String[] nodes;
+ List<String> issuedRequestNodes = issuedRequest.getNodes();
+ if (issuedRequestNodes != null) {
+ nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
+ } else {
+ nodes = null;
+ }
+
+
+ AMRMClient.ContainerRequest newRequest =
+ new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
+ nodes,
+ null,
+ pri,
+ true,
+ issuedRequest.getNodeLabelExpression());
+ issuedRequest = newRequest;
+ return issuedRequest;
+ }
+
/**
* Mark the request as completed (or canceled).
*/
@@ -202,7 +263,7 @@ public final class OutstandingRequest {
new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
sb.append(", node='").append(node).append('\'');
- sb.append(", requestedTime=").append(requestedTime);
+ sb.append(", requestedTime=").append(requestedTimeMillis);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index f6dc2de..e197a86 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -34,6 +34,11 @@ import java.util.Map;
/**
* Tracks outstanding requests made with a specific placement option.
* <p>
+ * <ol>
+ * <li>Used to decide when to return a node to 'can request containers here' list</li>
+ * <li>Used to identify requests where placement has timed out, and so issue relaxed requests</li>
+ * </ol>
+ * <p>
* If an allocation comes in that is not in the map: either the allocation
* was unplaced, or the placed allocation could not be met on the specified
* host, and the RM/scheduler fell back to another location.
@@ -56,7 +61,7 @@ public class OutstandingRequestTracker {
* @param role role index
* @return a new request
*/
- public synchronized OutstandingRequest addRequest(NodeInstance instance, int role) {
+ public synchronized OutstandingRequest newRequest(NodeInstance instance, int role) {
OutstandingRequest request =
new OutstandingRequest(role, instance);
if (request.isLocated()) {
@@ -69,7 +74,7 @@ public class OutstandingRequestTracker {
* Look up any oustanding request to a (role, hostname).
* @param role role index
* @param hostname hostname
- * @return the request or null if there was no outstanding one
+ * @return the request or null if there was no outstanding one in the {@link #placedRequests}
*/
public synchronized OutstandingRequest lookup(int role, String hostname) {
return placedRequests.get(new OutstandingRequest(role, hostname));
@@ -78,7 +83,7 @@ public class OutstandingRequestTracker {
/**
* Remove a request
* @param request matching request to find
- * @return the request
+ * @return the request or null for no match in the {@link #placedRequests}
*/
public synchronized OutstandingRequest remove(OutstandingRequest request) {
return placedRequests.remove(request);
@@ -86,10 +91,10 @@ public class OutstandingRequestTracker {
/**
* Notification that a container has been allocated -drop it
- * from the list of outstanding roles if need be
+ * from the {@link #placedRequests} structure.
* @param role role index
* @param hostname hostname
- * @return true if an entry was found and dropped
+ * @return true if an entry was found and removed
*/
public synchronized boolean onContainerAllocated(int role, String hostname) {
OutstandingRequest request =
@@ -167,21 +172,21 @@ public class OutstandingRequestTracker {
* in container assignments if more come back than expected
* @param rh RoleHistory instance
* @param inAllocated the list of allocated containers
- * @param outRequested initially empty list of requested locations
- * @param outUnrequested initially empty list of unrequested hosts
+ * @param outPlaceRequested initially empty list of requested locations
+ * @param outUnplaced initially empty list of unrequested hosts
*/
public synchronized void partitionRequests(RoleHistory rh,
List<Container> inAllocated,
- List<Container> outRequested,
- List<Container> outUnrequested) {
+ List<Container> outPlaceRequested,
+ List<Container> outUnplaced) {
Collections.sort(inAllocated, new newerThan(rh));
for (Container container : inAllocated) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
if (placedRequests.containsKey(new OutstandingRequest(role, hostname))) {
- outRequested.add(container);
+ outPlaceRequested.add(container);
} else {
- outUnrequested.add(container);
+ outUnplaced.add(container);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 632551e..0508579 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -47,7 +47,7 @@ public class ProviderAppState implements StateAccessForProviders {
private final Map<String, PublishedConfigSet> publishedConfigSets =
- new ConcurrentHashMap<String, PublishedConfigSet>(5);
+ new ConcurrentHashMap<>(5);
private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet();
private static final PatternValidator validator = new PatternValidator(
RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
@@ -106,7 +106,7 @@ public class ProviderAppState implements StateAccessForProviders {
public List<String> listConfigSets() {
synchronized (publishedConfigSets) {
- List<String> sets = new ArrayList<String>(publishedConfigSets.keySet());
+ List<String> sets = new ArrayList<>(publishedConfigSets.keySet());
return sets;
}
}
@@ -266,7 +266,7 @@ public class ProviderAppState implements StateAccessForProviders {
RoleStatus roleStatus = lookupRoleStatus(component);
List<RoleInstance> ownedContainerList = cloneOwnedContainerList();
List<RoleInstance> matching =
- new ArrayList<RoleInstance>(ownedContainerList.size());
+ new ArrayList<>(ownedContainerList.size());
int roleId = roleStatus.getPriority();
for (RoleInstance instance : ownedContainerList) {
if (instance.roleId == roleId) {
@@ -281,7 +281,7 @@ public class ProviderAppState implements StateAccessForProviders {
RoleStatus roleStatus = lookupRoleStatus(component);
ComponentInformation info = roleStatus.serialize();
List<RoleInstance> containers = lookupRoleContainers(component);
- info.containers = new ArrayList<String>(containers.size());
+ info.containers = new ArrayList<>(containers.size());
for (RoleInstance container : containers) {
info.containers.add(container.id);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index eef2b8f..acfe606 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -94,10 +94,8 @@ public class RoleHistory {
* Track the failed nodes. Currently used to make wiser decision of container
* ask with/without locality. Has other potential uses as well.
*/
- private Set<String> failedNodes = new HashSet<String>();
-
- // dummy to be used in maps for faster lookup where we don't care about values
- private final Object DUMMY_VALUE = new Object();
+ private Set<String> failedNodes = new HashSet<>();
+
public RoleHistory(List<ProviderRole> providerRoles) throws
BadConfigException {
@@ -117,7 +115,7 @@ public class RoleHistory {
outstandingRequests = new OutstandingRequestTracker();
- Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
+ Map<Integer, RoleStatus> roleStats = new HashMap<>();
for (ProviderRole providerRole : providerRoles) {
checkProviderRole(roleStats, providerRole);
}
@@ -156,7 +154,7 @@ public class RoleHistory {
throws BadConfigException {
log.debug("Validating/adding new provider role to role history: {} ",
providerRole);
- Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
+ Map<Integer, RoleStatus> roleStats = new HashMap<>();
for (ProviderRole role : providerRoles) {
roleStats.put(role.id, new RoleStatus(role));
@@ -171,7 +169,7 @@ public class RoleHistory {
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
- availableNodes = new HashMap<Integer, LinkedList<NodeInstance>>(roleSize);
+ availableNodes = new HashMap<>(roleSize);
}
/**
@@ -279,8 +277,10 @@ public class RoleHistory {
}
/**
- * Garbage collect the structure -this will dropp
- * all nodes that have been inactive since the (relative) age
+ * Garbage collect the structure -this will drop
+ * all nodes that have been inactive since the (relative) age.
+ * This will drop the failure counts of the nodes too, so it will
+ * lose information that matters.
* @param age relative age
*/
public void gc(long age) {
@@ -423,8 +423,7 @@ public class RoleHistory {
public synchronized void buildAvailableNodeLists() {
resetAvailableNodeLists();
// build the list of available nodes
- for (Map.Entry<String, NodeInstance> entry : nodemap
- .entrySet()) {
+ for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
NodeInstance ni = entry.getValue();
for (int i = 0; i < roleSize; i++) {
NodeEntry nodeEntry = ni.get(i);
@@ -458,13 +457,12 @@ public class RoleHistory {
private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
LinkedList<NodeInstance> instances = availableNodes.get(id);
if (instances == null) {
- instances = new LinkedList<NodeInstance>();
+ instances = new LinkedList<>();
availableNodes.put(id, instances);
}
return instances;
}
-
/**
* Sort an available node list
* @param role role to sort
@@ -472,7 +470,7 @@ public class RoleHistory {
private void sortAvailableNodeList(int role) {
List<NodeInstance> nodesForRoleId = getNodesForRoleId(role);
if (nodesForRoleId != null) {
- Collections.sort(nodesForRoleId, new NodeInstance.newerThan(role));
+ Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role));
}
}
@@ -492,7 +490,7 @@ public class RoleHistory {
List<NodeInstance> targets = getNodesForRoleId(roleKey);
if (targets == null) {
// add an empty list here for ease downstream
- targets = new ArrayList<NodeInstance>(0);
+ targets = new ArrayList<>(0);
}
int cnt = targets.size();
log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
@@ -527,7 +525,7 @@ public class RoleHistory {
*/
public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
NodeInstance node, RoleStatus role, Resource resource, String labelExpression) {
- OutstandingRequest outstanding = outstandingRequests.addRequest(node, role.getKey());
+ OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
return outstanding.buildContainerRequest(resource, role, now(), labelExpression);
}
@@ -622,9 +620,9 @@ public class RoleHistory {
//partition into requested and unrequested
List<Container> requested =
- new ArrayList<Container>(allocatedContainers.size());
+ new ArrayList<>(allocatedContainers.size());
List<Container> unrequested =
- new ArrayList<Container>(allocatedContainers.size());
+ new ArrayList<>(allocatedContainers.size());
outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
//give the unrequested ones lower priority
@@ -647,12 +645,13 @@ public class RoleHistory {
boolean requestFound =
outstandingRequests.onContainerAllocated(role, hostname);
if (desiredCount <= actualCount) {
- //cancel the nodes
+ // all oustanding requests have been satisfied
+ // tag nodes as available
List<NodeInstance>
hosts = outstandingRequests.cancelOutstandingRequests(role);
if (!hosts.isEmpty()) {
//add the list
- log.debug("Adding {} hosts for role {}", hosts.size(), role);
+ log.info("Adding {} hosts for role {}", hosts.size(), role);
nodeInstances.addAll(hosts);
sortAvailableNodeList(role);
}
@@ -708,8 +707,9 @@ public class RoleHistory {
*/
public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
for (NodeReport updatedNode : updatedNodes) {
- String hostname = updatedNode.getNodeId() == null ? null : updatedNode
- .getNodeId().getHost();
+ String hostname = updatedNode.getNodeId() == null
+ ? null
+ : updatedNode.getNodeId().getHost();
if (hostname == null) {
continue;
}
@@ -735,11 +735,10 @@ public class RoleHistory {
/**
* App state notified of a container completed
* @param container completed container
- * @param wasReleased
* @return true if the node was queued
*/
- public boolean onReleaseCompleted(Container container, boolean wasReleased) {
- return markContainerFinished(container, wasReleased, false);
+ public boolean onReleaseCompleted(Container container) {
+ return markContainerFinished(container, true, false);
}
/**
@@ -832,7 +831,7 @@ public class RoleHistory {
*/
@VisibleForTesting
public List<NodeInstance> cloneAvailableList(int role) {
- return new LinkedList<NodeInstance>(getOrCreateNodesForRoleId(role));
+ return new LinkedList<>(getOrCreateNodesForRoleId(role));
}
/**
@@ -850,7 +849,7 @@ public class RoleHistory {
* @return the list
*/
public List<String> cloneFailedNodes() {
- List<String> lst = new ArrayList<String>();
+ List<String> lst = new ArrayList<>();
lst.addAll(failedNodes);
return lst;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index ac1b131..7ab7dbe 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -74,6 +74,10 @@ public final class RoleStatus implements Cloneable {
return providerRole.placementPolicy;
}
+ public long getPlacementTimeoutSeconds() {
+ return providerRole.placementTimeoutSeconds;
+ }
+
/**
* The number of failures on a specific node that can be tolerated
* before selecting a different node for placement
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java b/slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
new file mode 100644
index 0000000..2e049cb
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/NewerFilesFirst.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.avro;
+
+import org.apache.hadoop.fs.Path;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Compare two filenames by name; the more recent one comes first
+ */
+public class NewerFilesFirst implements Comparator<Path>, Serializable {
+
+ /**
+ * Takes the ordering of path names from the normal string comparison
+ * and negates it, so that names that come after other names in
+ * the string sort come before here
+ * @param o1 leftmost
+ * @param o2 rightmost
+ * @return positive if o1 > o2
+ */
+ @Override
+ public int compare(Path o1, Path o2) {
+ return (o2.getName().compareTo(o1.getName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java b/slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
new file mode 100644
index 0000000..407aaa6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/OlderFilesFirst.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.avro;
+
+import org.apache.hadoop.fs.Path;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Compare two filenames by name; the older ones comes first
+ */
+public class OlderFilesFirst implements Comparator<Path>, Serializable {
+
+ /**
+ * Takes the ordering of path names from the normal string comparison
+ * and negates it, so that names that come after other names in
+ * the string sort come before here
+ * @param o1 leftmost
+ * @param o2 rightmost
+ * @return positive if o1 > o2
+ */
+ @Override
+ public int compare(Path o1, Path o2) {
+ return (o1.getName().compareTo(o2.getName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index 422ffeb..031632b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -52,11 +52,9 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
@@ -90,7 +88,7 @@ public class RoleHistoryWriter {
throws IOException {
try {
DatumWriter<RoleHistoryRecord> writer =
- new SpecificDatumWriter<RoleHistoryRecord>(RoleHistoryRecord.class);
+ new SpecificDatumWriter<>(RoleHistoryRecord.class);
int roles = history.getRoleSize();
RoleHistoryHeader header = new RoleHistoryHeader();
@@ -184,7 +182,7 @@ public class RoleHistoryWriter {
BadConfigException {
try {
DatumReader<RoleHistoryRecord> reader =
- new SpecificDatumReader<RoleHistoryRecord>(RoleHistoryRecord.class);
+ new SpecificDatumReader<>(RoleHistoryRecord.class);
Decoder decoder =
DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
in);
@@ -351,16 +349,15 @@ public class RoleHistoryWriter {
public static void sortHistoryPaths(List<Path> paths) {
Collections.sort(paths, new NewerFilesFirst());
}
-
-
+
/**
* Iterate through the paths until one can be loaded
* @param roleHistory role history
* @param paths paths to load
* @return the path of any loaded history -or null if all failed to load
*/
- public Path attemptToReadHistory(RoleHistory roleHistory, FileSystem fileSystem, List<Path> paths) throws
- BadConfigException {
+ public Path attemptToReadHistory(RoleHistory roleHistory, FileSystem fileSystem, List<Path> paths)
+ throws BadConfigException {
ListIterator<Path> pathIterator = paths.listIterator();
boolean success = false;
Path path = null;
@@ -389,9 +386,8 @@ public class RoleHistoryWriter {
* @throws IOException if indexing the history directory fails.
*/
public Path loadFromHistoryDir(FileSystem fs, Path dir,
- RoleHistory roleHistory) throws
- IOException,
- BadConfigException {
+ RoleHistory roleHistory)
+ throws IOException, BadConfigException {
assert fs != null: "null filesystem";
List<Path> entries = findAllHistoryEntries(fs, dir, false);
return attemptToReadHistory(roleHistory, fs, entries);
@@ -407,9 +403,8 @@ public class RoleHistoryWriter {
* check to stop the entire dir being purged)
* @throws IOException IO problems
*/
- public int purgeOlderHistoryEntries(FileSystem fileSystem, Path keep) throws
- IOException {
- assert fileSystem != null : "null filesystem";
+ public int purgeOlderHistoryEntries(FileSystem fileSystem, Path keep)
+ throws IOException { assert fileSystem != null : "null filesystem";
if (!fileSystem.exists(keep)) {
throw new FileNotFoundException(keep.toString());
}
@@ -429,44 +424,5 @@ public class RoleHistoryWriter {
}
return deleteCount;
}
-
- /**
- * Compare two filenames by name; the more recent one comes first
- */
- public static class NewerFilesFirst implements Comparator<Path> ,
- Serializable {
- /**
- * Takes the ordering of path names from the normal string comparison
- * and negates it, so that names that come after other names in
- * the string sort come before here
- * @param o1 leftmost
- * @param o2 rightmost
- * @return positive if o1 > o2
- */
- @Override
- public int compare(Path o1, Path o2) {
- return (o2.getName().compareTo(o1.getName()));
- }
- }
- /**
- * Compare two filenames by name; the older ones comes first
- */
- public static class OlderFilesFirst implements Comparator<Path> ,
- Serializable {
-
- /**
- * Takes the ordering of path names from the normal string comparison
- * and negates it, so that names that come after other names in
- * the string sort come before here
- * @param o1 leftmost
- * @param o2 rightmost
- * @return positive if o1 > o2
- */
- @Override
- public int compare(Path o1, Path o2) {
- return (o1.getName().compareTo(o2.getName()));
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index dbb70fa..c7a38f5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -18,8 +18,6 @@
package org.apache.slider.server.appmaster.model.history
-import java.util.List;
-
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
@@ -164,7 +162,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.active == 0
// release completed
- roleHistory.onReleaseCompleted(container, true)
+ roleHistory.onReleaseCompleted(container)
assert roleEntry.releasing == 0
assert roleEntry.live == 0
assert roleEntry.active == 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
index 612cce8..74dfd42 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster.model.history
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.state.NodeInstance
+import org.junit.Before
import org.junit.Test
/**
@@ -33,9 +34,17 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+ NodeInstance age6failing = nodeInstance(1006, 0, 0, 0)
+ NodeInstance age1failing = nodeInstance(1000, 0, 0, 0)
List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ @Before
+ public void setup() {
+ age6failing.get(0).failedRecently = 2;
+ age1failing.get(0).failedRecently = 1;
+ }
+
@Override
String getTestName() {
return "TestNIComparators"
@@ -44,16 +53,24 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
@Test
public void testNewerThan() throws Throwable {
- Collections.sort(nodes, new NodeInstance.newerThan(0))
+ Collections.sort(nodes, new NodeInstance.Preferred(0))
assertListEquals(nodes,
[age4Active1, age3Active0, age2Active2, age1Active4])
}
@Test
+ public void testFailureCountFirst() throws Throwable {
+ def preferred = new NodeInstance.Preferred(0)
+ assert preferred.compare(age6failing, age1failing) == -1
+ assert preferred.compare(age1failing, age6failing) == 1
+ assert preferred.compare(age1failing, age1failing) == 0
+ }
+
+ @Test
public void testNewerThanNoRole() throws Throwable {
nodes << empty
- Collections.sort(nodes, new NodeInstance.newerThan(0))
+ Collections.sort(nodes, new NodeInstance.Preferred(0))
assertListEquals(nodes,
[age4Active1, age3Active0, age2Active2, age1Active4, empty])
}
@@ -61,7 +78,7 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
@Test
public void testMoreActiveThan() throws Throwable {
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
assertListEquals(nodes,
[age1Active4, age2Active2, age4Active1, age3Active0],)
}
@@ -69,7 +86,7 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
@Test
public void testMoreActiveThanEmpty() throws Throwable {
nodes << empty
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
assertListEquals(nodes,
[age1Active4, age2Active2, age4Active1, age3Active0, empty])
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 7085678..79a9bd6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -38,7 +38,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testAddRetrieveEntry() throws Throwable {
- OutstandingRequest request = tracker.addRequest(host1, 0)
+ OutstandingRequest request = tracker.newRequest(host1, 0)
assert tracker.lookup(0, "host1").equals(request)
assert tracker.remove(request).equals(request)
assert !tracker.lookup(0, "host1")
@@ -46,9 +46,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testAddCompleteEntry() throws Throwable {
- tracker.addRequest(host1, 0)
- tracker.addRequest(host2, 0)
- tracker.addRequest(host1, 1)
+ tracker.newRequest(host1, 0)
+ tracker.newRequest(host2, 0)
+ tracker.newRequest(host1, 1)
assert tracker.onContainerAllocated(1, "host1")
assert !tracker.lookup(1, "host1")
assert tracker.lookup(0, "host1")
@@ -56,9 +56,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testCancelEntries() throws Throwable {
- OutstandingRequest r1 = tracker.addRequest(host1, 0)
- OutstandingRequest r2 = tracker.addRequest(host2, 0)
- OutstandingRequest r3 = tracker.addRequest(host1, 1)
+ OutstandingRequest r1 = tracker.newRequest(host1, 0)
+ OutstandingRequest r2 = tracker.newRequest(host2, 0)
+ OutstandingRequest r3 = tracker.newRequest(host1, 1)
List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
assert canceled.size() == 2
assert canceled.contains(host1)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/559ed00a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
index a0663e8..aef22fb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
@@ -26,6 +26,7 @@ import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.NewerFilesFirst
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -119,7 +120,7 @@ class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
@Test
public void testPathnameComparator() throws Throwable {
- def newerName = new RoleHistoryWriter.NewerFilesFirst()
+ def newerName = new NewerFilesFirst()
log.info("$h_5fffa name is ${h_5fffa.getName()}")
log.info("$h_0406c name is ${h_0406c.getName()}")
[10/20] incubator-slider git commit: SLIDER-799 action
EscalateOutstandingRequests
Posted by st...@apache.org.
SLIDER-799 action EscalateOutstandingRequests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/06e8db3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/06e8db3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/06e8db3e
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 06e8db3e8fb00f299333604491f7c29debac5955
Parents: 8338898
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 17 11:23:46 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 17 11:23:46 2015 +0000
----------------------------------------------------------------------
.../actions/EscalateOutstandingRequests.java | 25 +++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06e8db3e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
index e527be6..2c545ea 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
@@ -18,5 +18,28 @@
package org.apache.slider.server.appmaster.actions;
-public class EscalateOutstandingRequests {
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Escalate outstanding requests by asking AM
+ */
+public class EscalateOutstandingRequests extends AsyncAction {
+
+ public EscalateOutstandingRequests() {
+ super("EscalateOutstandingRequests");
+ }
+
+ public EscalateOutstandingRequests(long delay,
+ TimeUnit timeUnit) {
+ super("EscalateOutstandingRequests", delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws
+ Exception {
+ appMaster.escalateOutstandingRequests();
+ }
}
[18/20] incubator-slider git commit: SLIDER--799 SLIDER-828 when
containers are allocated, explicitly cancel the request
Posted by st...@apache.org.
SLIDER--799 SLIDER-828 when containers are allocated, explicitly cancel the request
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1938323c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1938323c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1938323c
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 1938323c7fef2666a429a06a55208a528e93f64b
Parents: 7d9a9e9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 24 13:44:09 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 24 13:49:14 2015 +0000
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 2 +-
.../appmaster/management/MetricsConstants.java | 33 ++++-
.../operations/AsyncRMOperationHandler.java | 6 +-
.../operations/CancelRequestOperation.java | 14 +-
.../slider/server/appmaster/state/AppState.java | 147 +++++++++----------
.../server/appmaster/state/RoleHistory.java | 2 +-
.../TestMockAppStateRebuildOnAMRestart.groovy | 4 +-
7 files changed, 124 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index ab6b55c..b8584f7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1840,7 +1840,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
rmOperationHandler.cancelSingleRequest(request);
}
- /* =================================================================== */
+/* =================================================================== */
/* END */
/* =================================================================== */
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
index e55cf60..31a82a3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
@@ -22,6 +22,35 @@ package org.apache.slider.server.appmaster.management;
* Constants used in slider for metrics registration and lookup
*/
public class MetricsConstants {
- public static final String CONTAINERS_OUTSTANDING_REQUESTS =
- "containers.outstanding-requests";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_OUTSTANDING_REQUESTS = "containers.outstanding-requests";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_STARTED = "containers.started";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_SURPLUS = "containers.surplus";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_COMPLETED = "containers.completed";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_FAILED = "containers.failed";
+
+ /**
+ * {@value}
+ */
+ public static final String CONTAINERS_START_FAILED = "containers.start-failed";
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
index 7c98551..11afc0e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
@@ -50,8 +50,10 @@ public class AsyncRMOperationHandler extends RMOperationHandler {
// need to revoke a previously issued container request
// so enum the sets and pick some
int remaining = cancelSinglePriorityRequests(priority1, count);
- remaining = cancelSinglePriorityRequests(priority2, remaining);
-
+ if (priority2 != null) {
+ remaining = cancelSinglePriorityRequests(priority2, remaining);
+ }
+
return remaining;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
index 9e9f277..754bf28 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
@@ -18,11 +18,12 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.server.appmaster.state.ContainerPriority;
/**
- * Cancel a container request
+ * Cancel a container request at the given priority/proirities.
*/
public class CancelRequestOperation extends AbstractRMOperation {
@@ -30,7 +31,15 @@ public class CancelRequestOperation extends AbstractRMOperation {
private final Priority priority2;
private final int count;
+ /**
+ * Create an instance
+ * @param priority1 first priority, the one that is released first
+ * @param priority2 optional second priority
+ * @param count number of requests to cancel
+ */
public CancelRequestOperation(Priority priority1, Priority priority2, int count) {
+ Preconditions.checkArgument(priority1 != null, "null priority");
+ Preconditions.checkArgument(count >= 0, "negative count");
this.priority1 = priority1;
this.priority2 = priority2;
this.count = count;
@@ -45,7 +54,8 @@ public class CancelRequestOperation extends AbstractRMOperation {
public String toString() {
return "release " + count
+ " requests for " + ContainerPriority.toString(priority1)
- + " and " + ContainerPriority.toString(priority2);
+ + (priority2 != null ?
+ (" and " + ContainerPriority.toString(priority2)) : "");
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 20e2fc0..34b0492 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -170,7 +170,7 @@ public class AppState {
* Client properties created via the provider -static for the life
* of the application
*/
- private Map<String, String> clientProperties = new HashMap<String, String>();
+ private Map<String, String> clientProperties = new HashMap<>();
/**
* This is a template of the cluster status
@@ -196,7 +196,7 @@ public class AppState {
* been allocated but are not live; it is a superset of the live list
*/
private final ConcurrentMap<ContainerId, RoleInstance> ownedContainers =
- new ConcurrentHashMap<ContainerId, RoleInstance>();
+ new ConcurrentHashMap<>();
/**
* Hash map of the containers we have released, but we
@@ -204,33 +204,33 @@ public class AppState {
* containers is treated as a successful outcome
*/
private final ConcurrentMap<ContainerId, Container> containersBeingReleased =
- new ConcurrentHashMap<ContainerId, Container>();
+ new ConcurrentHashMap<>();
/**
* Counter for completed containers ( complete denotes successful or failed )
*/
- private final AtomicInteger completedContainerCount = new AtomicInteger();
+ private final Counter completedContainerCount = new Counter();
/**
* Count of failed containers
*/
- private final AtomicInteger failedContainerCount = new AtomicInteger();
+ private final Counter failedContainerCount = new Counter();
/**
* # of started containers
*/
- private final AtomicInteger startedContainers = new AtomicInteger();
+ private final Counter startedContainers = new Counter();
/**
* # of containers that failed to start
*/
- private final AtomicInteger startFailedContainers = new AtomicInteger();
+ private final Counter startFailedContainerCount = new Counter();
/**
* Track the number of surplus containers received and discarded
*/
- private final AtomicInteger surplusContainers = new AtomicInteger();
+ private final Counter surplusContainers = new Counter();
/**
@@ -244,21 +244,21 @@ public class AppState {
* the node is promoted from here to the containerMap
*/
private final Map<ContainerId, RoleInstance> startingNodes =
- new ConcurrentHashMap<ContainerId, RoleInstance>();
+ new ConcurrentHashMap<>();
/**
* List of completed nodes. This isn't kept in the CD as it gets too
* big for the RPC responses. Indeed, we should think about how deep to get this
*/
private final Map<ContainerId, RoleInstance> completedNodes
- = new ConcurrentHashMap<ContainerId, RoleInstance>();
+ = new ConcurrentHashMap<>();
/**
* Nodes that failed to start.
* Again, kept out of the CD
*/
private final Map<ContainerId, RoleInstance> failedNodes =
- new ConcurrentHashMap<ContainerId, RoleInstance>();
+ new ConcurrentHashMap<>();
/**
* Nodes that came assigned to a role above that
@@ -267,11 +267,11 @@ public class AppState {
private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
/**
- * Map of containerID -> cluster nodes, for status reports.
+ * Map of containerID to cluster nodes, for status reports.
* Access to this should be synchronized on the clusterDescription
*/
private final Map<ContainerId, RoleInstance> liveNodes =
- new ConcurrentHashMap<ContainerId, RoleInstance>();
+ new ConcurrentHashMap<>();
private final AtomicInteger completionOfNodeNotInLiveListEvent =
new AtomicInteger();
private final AtomicInteger completionOfUnknownContainerEvent =
@@ -311,54 +311,53 @@ public class AppState {
MetricsAndMonitoring metricsAndMonitoring) {
this.recordFactory = recordFactory;
this.metricsAndMonitoring = metricsAndMonitoring;
-
+
// register any metrics
- MetricRegistry metrics = metricsAndMonitoring.getMetrics();
- metrics.register(
+ register(MetricsConstants.CONTAINERS_OUTSTANDING_REQUESTS, outstandingContainerRequests);
+ register(MetricsConstants.CONTAINERS_SURPLUS, surplusContainers);
+ register(MetricsConstants.CONTAINERS_STARTED, startedContainers);
+ register(MetricsConstants.CONTAINERS_COMPLETED, completedContainerCount);
+ register(MetricsConstants.CONTAINERS_FAILED, failedContainerCount);
+ register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount);
+ }
+
+ private void register(String name, Counter counter) {
+ this.metricsAndMonitoring.getMetrics().register(
MetricRegistry.name(AppState.class,
- MetricsConstants.CONTAINERS_OUTSTANDING_REQUESTS),
- outstandingContainerRequests);
+ name), counter);
}
- public int getFailedCountainerCount() {
- return failedContainerCount.get();
+ public long getFailedCountainerCount() {
+ return failedContainerCount.getCount();
}
/**
- * Increment the count and return the new value
- * @return the latest failed container count
+ * Increment the count
*/
- public int incFailedCountainerCount() {
- return failedContainerCount.incrementAndGet();
+ public void incFailedCountainerCount() {
+ failedContainerCount.inc();
}
- public int getStartFailedCountainerCount() {
- return startFailedContainers.get();
+ public long getStartFailedCountainerCount() {
+ return startFailedContainerCount.getCount();
}
/**
* Increment the count and return the new value
- * @return the latest failed container count
*/
- public int incStartedCountainerCount() {
- return startedContainers.incrementAndGet();
+ public void incStartedCountainerCount() {
+ startedContainers.inc();
}
- public int getStartedCountainerCount() {
- return startedContainers.get();
+ public long getStartedCountainerCount() {
+ return startedContainers.getCount();
}
/**
* Increment the count and return the new value
- * @return the latest failed container count
*/
- public int incStartFailedCountainerCount() {
- return startFailedContainers.incrementAndGet();
- }
-
-
- public AtomicInteger getStartFailedContainers() {
- return startFailedContainers;
+ public void incStartFailedCountainerCount() {
+ startFailedContainerCount.inc();
}
public AtomicInteger getCompletionOfNodeNotInLiveListEvent() {
@@ -535,7 +534,7 @@ public class AppState {
this.applicationInfo = applicationInfo != null ? applicationInfo
: new HashMap<String, String>();
- clientProperties = new HashMap<String, String>();
+ clientProperties = new HashMap<>();
containerReleaseSelector = releaseSelector;
@@ -1744,19 +1743,19 @@ public class AppState {
* keylist.
*/
protected Map<String, Integer> getLiveStatistics() {
- Map<String, Integer> sliderstats = new HashMap<String, Integer>();
+ Map<String, Integer> sliderstats = new HashMap<>();
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE,
+ liveNodes.size());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
- completedContainerCount.get());
+ (int)completedContainerCount.getCount());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
- failedContainerCount.get());
- sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE,
- liveNodes.size());
+ (int)failedContainerCount.getCount());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED,
- startedContainers.get());
+ (int)startedContainers.getCount());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED,
- startFailedContainers.get());
+ (int) startFailedContainerCount.getCount());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS,
- surplusContainers.get());
+ (int)surplusContainers.getCount());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED,
completionOfUnknownContainerEvent.get());
return sliderstats;
@@ -1893,7 +1892,6 @@ public class AppState {
throws SliderInternalStateException, TriggerClusterTeardownException {
List<AbstractRMOperation> operations = new ArrayList<>();
int delta;
- String details;
int expected;
String name = role.getName();
synchronized (role) {
@@ -1903,7 +1901,7 @@ public class AppState {
log.info("Reviewing {} : expected {}", role, expected);
checkFailureThreshold(role);
-
+
if (expected < 0 ) {
// negative value: fail
throw new TriggerClusterTeardownException(
@@ -1912,7 +1910,7 @@ public class AppState {
"Negative component count of %d desired for component %s",
expected, role);
}
-
+
if (delta > 0) {
log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
delta, expected);
@@ -2004,7 +2002,7 @@ public class AppState {
for (RoleInstance possible : finalCandidates) {
log.debug("Targeting for release: {}", possible);
containerReleaseSubmitted(possible.container);
- operations.add(new ContainerReleaseOperation(possible.getId()));
+ operations.add(new ContainerReleaseOperation(possible.getId()));
}
}
@@ -2033,7 +2031,6 @@ public class AppState {
return operations;
}
-
/**
* Find a container running on a specific host -looking
* into the node ID to determine this.
@@ -2055,7 +2052,7 @@ public class AppState {
}
return null;
}
-
+
/**
* Release all containers.
* @return a list of operations to execute
@@ -2105,40 +2102,42 @@ public class AppState {
String containerHostInfo = container.getNodeId().getHost()
+ ":" +
container.getNodeId().getPort();
- int allocated;
- int desired;
//get the role
- ContainerId cid = container.getId();
- RoleStatus role = lookupRoleStatus(container);
+ final ContainerId cid = container.getId();
+ final RoleStatus role = lookupRoleStatus(container);
//dec requested count
decrementRequestCount(role);
+
+ // cancel an allocation request which granted this, so as to avoid repeated
+ // requests
+ releaseOperations.add(new CancelRequestOperation(container.getPriority(), null, 1));
+
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
- allocated = role.incActual();
+ final int allocated = role.incActual();
+ final int desired = role.getDesired();
- //look for (race condition) where we get more back than we asked
- desired = role.getDesired();
-
- ContainerAllocationOutcome outcome = roleHistory.onContainerAllocated(container,
- desired,
- allocated);
+ final String roleName = role.getName();
+ final ContainerAllocationOutcome outcome =
+ roleHistory.onContainerAllocated(container, desired, allocated);
+ //look for condition where we get more back than we asked
if (allocated > desired) {
- log.info("Discarding surplus container {} on {}", cid,
- containerHostInfo);
+ log.info("Discarding surplus {} container {} on {}", roleName, cid,
+ containerHostInfo);
releaseOperations.add(new ContainerReleaseOperation(cid));
//register as a surplus node
surplusNodes.add(cid);
- surplusContainers.incrementAndGet();
+ surplusContainers.inc();
//and, as we aren't binding it to role, dec that role's actual count
role.decActual();
} else {
- // this is valid, so decrement the number of outstanding requests
+ // Allocation being accepted -so decrement the number of outstanding requests
decOutstandingContainerRequests();
- String roleName = role.getName();
+
log.info("Assigning role {} to container" +
" {}," +
" on {}:{},",
@@ -2174,8 +2173,8 @@ public class AppState {
* @return true if a rebuild took place (even if size 0)
* @throws RuntimeException on problems
*/
- private boolean rebuildModelFromRestart(List<Container> liveContainers) throws
- BadClusterStateException {
+ private boolean rebuildModelFromRestart(List<Container> liveContainers)
+ throws BadClusterStateException {
if (liveContainers == null) {
return false;
}
@@ -2193,8 +2192,8 @@ public class AppState {
* @param container container that was running before the AM restarted
* @throws RuntimeException on problems
*/
- private void addRestartedContainer(Container container) throws
- BadClusterStateException {
+ private void addRestartedContainer(Container container)
+ throws BadClusterStateException {
String containerHostInfo = container.getNodeId().getHost()
+ ":" +
container.getNodeId().getPort();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 99108fe..64f9184 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -625,7 +625,7 @@ public class RoleHistory {
List<Container> unrequested =
new ArrayList<>(allocatedContainers.size());
outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
-
+
//give the unrequested ones lower priority
requested.addAll(unrequested);
return requested;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1938323c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index e0fdf1b..c310583 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -85,9 +85,9 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
null,
new MostRecentContainerReleaseSelector())
- assert appState.getStartedCountainerCount() == clusterSize
+ assert appState.startedCountainerCount == clusterSize
- appState.getRoleHistory().dump();
+ appState.roleHistory.dump();
//check that the app state direct structures match
List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0)
[09/20] incubator-slider git commit: SIDER-799 CancelSingleRequest
Posted by st...@apache.org.
SIDER-799 CancelSingleRequest
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/83388984
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/83388984
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/83388984
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 833889843cdbac907bc1a264d085c309e3be3c7a
Parents: 6d955af
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 17 11:23:14 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 17 11:23:14 2015 +0000
----------------------------------------------------------------------
.../operations/CancelSingleRequest.java | 27 +++++---------------
1 file changed, 7 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/83388984/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
index e4ccb10..3bb3b09 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -18,41 +18,28 @@
package org.apache.slider.server.appmaster.operations;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
/**
* Cancel a container request
*/
public class CancelSingleRequest extends AbstractRMOperation {
- private final Priority priority1;
- private final Priority priority2;
- private final int count;
+ private final AMRMClient.ContainerRequest request;
- public CancelSingleRequest(Priority priority1, Priority priority2, int count) {
- this.priority1 = priority1;
- this.priority2 = priority2;
- this.count = count;
+ public CancelSingleRequest(AMRMClient.ContainerRequest request) {
+ this.request = request;
}
@Override
public void execute(RMOperationHandlerActions handler) {
- handler.cancelContainerRequests(priority1, priority2, count);
+ handler.cancelSingleRequest(request);
}
@Override
public String toString() {
- return "release " + count
- + " requests for " + ContainerPriority.toString(priority1)
- + " and " + ContainerPriority.toString(priority2);
+ return "cancel single request for container at " + request.getPriority().toString();
}
- /**
- * Get the number to release
- * @return the number of containers to release
- */
- public int getCount() {
- return count;
- }
+
}
[20/20] incubator-slider git commit: SLIDER-799 SLIDER-817 request
tracker builds cancel operation from the resource used in the request...tests
updated to handle the changes
Posted by st...@apache.org.
SLIDER-799 SLIDER-817 request tracker builds cancel operation from the resource used in the request...tests updated to handle the changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b952b640
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b952b640
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b952b640
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: b952b6401a22c65053c85a6a1238f1928d3eb243
Parents: 43c61fb
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 24 19:33:36 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 24 19:33:36 2015 +0000
----------------------------------------------------------------------
.../operations/CancelRequestOperation.java | 68 ---------------
.../slider/server/appmaster/state/AppState.java | 48 +++++++----
.../state/ContainerAllocationOutcome.java | 15 ++++
.../appmaster/state/OutstandingRequest.java | 12 ++-
.../state/OutstandingRequestTracker.java | 87 ++++++++++++++++----
.../server/appmaster/state/RoleHistory.java | 30 ++++++-
.../appstate/TestMockAppStateFlexing.groovy | 20 ++++-
.../TestMockAppStateRMOperations.groovy | 62 ++++++++------
.../TestMockAppStateRolePlacement.groovy | 14 +++-
...tRoleHistoryOutstandingRequestTracker.groovy | 12 +--
.../TestRoleHistoryRequestTracking.groovy | 8 +-
.../appmaster/model/mock/Allocator.groovy | 1 +
.../appmaster/model/mock/MockYarnEngine.groovy | 2 +-
13 files changed, 236 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
deleted file mode 100644
index 754bf28..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.operations;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-
-/**
- * Cancel a container request at the given priority/proirities.
- */
-public class CancelRequestOperation extends AbstractRMOperation {
-
- private final Priority priority1;
- private final Priority priority2;
- private final int count;
-
- /**
- * Create an instance
- * @param priority1 first priority, the one that is released first
- * @param priority2 optional second priority
- * @param count number of requests to cancel
- */
- public CancelRequestOperation(Priority priority1, Priority priority2, int count) {
- Preconditions.checkArgument(priority1 != null, "null priority");
- Preconditions.checkArgument(count >= 0, "negative count");
- this.priority1 = priority1;
- this.priority2 = priority2;
- this.count = count;
- }
-
- @Override
- public void execute(RMOperationHandlerActions handler) {
- handler.cancelContainerRequests(priority1, priority2, count);
- }
-
- @Override
- public String toString() {
- return "release " + count
- + " requests for " + ContainerPriority.toString(priority1)
- + (priority2 != null ?
- (" and " + ContainerPriority.toString(priority2)) : "");
- }
-
- /**
- * Get the number to release
- * @return the number of containers to release
- */
- public int getCount() {
- return count;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 20c1792..c68b2a9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -68,7 +67,6 @@ import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.MetricsConstants;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelRequestOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.slf4j.Logger;
@@ -1941,25 +1939,33 @@ public class AppState {
if (outstandingRequests > 0) {
// outstanding requests.
int toCancel = Math.min(outstandingRequests, excess);
- Priority p1 =
- ContainerPriority.createPriority(role.getPriority(), true);
- Priority p2 =
- ContainerPriority.createPriority(role.getPriority(), false);
- // TODO Delegate to Role History
- operations.add(new CancelRequestOperation(p1, p2, toCancel));
+
+ // Delegate to Role History
+
+ List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel);
+ log.info("Found {} outstanding requests to cancel", cancellations.size());
+ operations.addAll(cancellations);
+ if (toCancel != cancellations.size()) {
+ log.error("Tracking of outstanding requests is not in sync with the summary statistics:" +
+ " expected to be able to cancel {} requests, but got {}",
+ toCancel, cancellations.size());
+ }
+
role.cancel(toCancel);
excess -= toCancel;
assert excess >= 0 : "Attempted to cancel too many requests";
log.info("Submitted {} cancellations, leaving {} to release",
toCancel, excess);
if (excess == 0) {
- log.info("After cancelling requests, application is at desired size");
+ log.info("After cancelling requests, application is now at desired size");
}
}
// after the cancellation there may be no excess
if (excess > 0) {
+
+ // there's an excess, so more to cancel
// get the nodes to release
int roleId = role.getKey();
@@ -1978,7 +1984,7 @@ public class AppState {
}
}
- // warn if the desired state can't be reaced
+ // warn if the desired state can't be reached
int numberAvailableForRelease = containersToRelease.size();
if (numberAvailableForRelease < excess) {
log.warn("Not enough containers to release, have {} and need {} more",
@@ -2001,7 +2007,7 @@ public class AppState {
// then build up a release operation, logging each container as released
for (RoleInstance possible : finalCandidates) {
- log.debug("Targeting for release: {}", possible);
+ log.info("Targeting for release: {}", possible);
containerReleaseSubmitted(possible.container);
operations.add(new ContainerReleaseOperation(possible.getId()));
}
@@ -2009,6 +2015,7 @@ public class AppState {
}
+ // list of operations to execute
return operations;
}
@@ -2111,18 +2118,27 @@ public class AppState {
//dec requested count
decrementRequestCount(role);
- // cancel an allocation request which granted this, so as to avoid repeated
- // requests
- releaseOperations.add(new CancelRequestOperation(container.getPriority(), null, 1));
-
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
final int allocated = role.incActual();
final int desired = role.getDesired();
final String roleName = role.getName();
- final ContainerAllocationOutcome outcome =
+ final ContainerAllocation allocation =
roleHistory.onContainerAllocated(container, desired, allocated);
+ final ContainerAllocationOutcome outcome = allocation.outcome;
+
+ // cancel an allocation request which granted this, so as to avoid repeated
+ // requests
+ if (allocation.origin != null && allocation.origin.getIssuedRequest() != null) {
+ releaseOperations.add(allocation.origin.createCancelOperation());
+ } else {
+ // there's a request, but no idea what to cancel.
+ // rather than try to recover from it inelegantly, (and cause more confusion),
+ // log the event, but otherwise continue
+ log.warn("Unexpected allocation of container "
+ + SliderUtils.containerToString(container));
+ }
//look for condition where we get more back than we asked
if (allocated > desired) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
index 6639300..5b3a93c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
@@ -22,8 +22,23 @@ package org.apache.slider.server.appmaster.state;
* Outcome of the assignment
*/
public enum ContainerAllocationOutcome {
+ /**
+ * There wasn't a request for this
+ */
Unallocated,
+
+ /**
+ * Open placement
+ */
Open,
+
+ /**
+ * Allocated explicitly where requested
+ */
Placed,
+
+ /**
+ * This was an escalated placement
+ */
Escalated
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 24946af..12b4b53 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -244,7 +245,7 @@ public final class OutstandingRequest {
* so as to place it into the relaxed list.
*/
public synchronized AMRMClient.ContainerRequest escalate() {
- Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued "+ this);
+ Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this);
escalated = true;
// this is now the priority
@@ -352,4 +353,13 @@ public final class OutstandingRequest {
sb.append('}');
return sb.toString();
}
+
+ /**
+ * Create a cancel operation
+ * @return an operation that can be used to cancel the request
+ */
+ public CancelSingleRequest createCancelOperation() {
+ Preconditions.checkState(issuedRequest!=null, "No issued request to cancel");
+ return new CancelSingleRequest(issuedRequest);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 05a8052..97d321c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -119,38 +119,44 @@ public class OutstandingRequestTracker {
* from the {@link #placedRequests} structure.
* @param role role index
* @param hostname hostname
- * @param resource
* @return the allocation outcome
*/
- public synchronized ContainerAllocationOutcome onContainerAllocated(int role,
+ public synchronized ContainerAllocation onContainerAllocated(int role,
String hostname,
Container container) {
+ final String containerDetails = SliderUtils.containerToString(container);
+ log.debug("Processing allocation for role {} on {}", role,
+ containerDetails);
+ ContainerAllocation allocation = new ContainerAllocation();
ContainerAllocationOutcome outcome;
OutstandingRequest request =
- placedRequests.remove(new OutstandingRequest(role, hostname));
+ placedRequests.remove(new OutstandingRequest(role, hostname));
if (request != null) {
//satisfied request
- log.info("Found placed request for container: {}", request);
+ log.debug("Found placed request for container: {}", request);
request.completed();
// derive outcome from status of tracked request
outcome = request.isEscalated()
- ? ContainerAllocationOutcome.Escalated
- : ContainerAllocationOutcome.Placed;
+ ? ContainerAllocationOutcome.Escalated
+ : ContainerAllocationOutcome.Placed;
} else {
// not in the list; this is an open placement
// scan through all containers in the open request list
request = removeOpenRequest(container);
if (request != null) {
- log.info("Found open request for container: {}", request);
+ log.debug("Found open request for container: {}", request);
request.completed();
outcome = ContainerAllocationOutcome.Open;
} else {
- log.warn("Container allocation was not expected :"
- + SliderUtils.containerToString(container));
+ log.warn("No open request found for container {}, outstanding queue has {} entries ",
+ containerDetails,
+ openRequests.size());
outcome = ContainerAllocationOutcome.Unallocated;
}
}
- return outcome;
+ allocation.origin = request;
+ allocation.outcome = outcome;
+ return allocation;
}
/**
@@ -167,11 +173,15 @@ public class OutstandingRequestTracker {
ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
while (openlist.hasNext() && request == null) {
OutstandingRequest r = openlist.next();
- if (r.getPriority() == pri
- && r.resourceRequirementsMatch(resource)) {
- // match of priority and resources
- request = r;
- openlist.remove();
+ if (r.getPriority() == pri) {
+ // matching resource
+ if (r.resourceRequirementsMatch(resource)) {
+ // match of priority and resources
+ request = r;
+ openlist.remove();
+ } else {
+ log.debug("Matched priorities but resources different");
+ }
}
}
return request;
@@ -314,6 +324,7 @@ public class OutstandingRequestTracker {
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public synchronized List<AbstractRMOperation> escalateOutstandingRequests(long now) {
if (placedRequests.isEmpty()) {
return NO_REQUESTS;
@@ -327,7 +338,7 @@ public class OutstandingRequestTracker {
if (outstandingRequest.shouldEscalate(now)) {
// time to escalate
- CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.getIssuedRequest());
+ CancelSingleRequest cancel = outstandingRequest.createCancelOperation();
operations.add(cancel);
AMRMClient.ContainerRequest escalated =
outstandingRequest.escalate();
@@ -338,4 +349,48 @@ public class OutstandingRequestTracker {
}
return operations;
}
+
+ /**
+ * Extract a specific number of open requests for a role
+ * @param roleId role Id
+ * @param count count to extract
+ * @return a list of requests which are no longer in the open request list
+ */
+ public synchronized List<OutstandingRequest> extractOpenRequestsForRole(int roleId, int count) {
+ List<OutstandingRequest> results = new ArrayList<>();
+ ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
+ while (openlist.hasNext() && count > 0) {
+ OutstandingRequest openRequest = openlist.next();
+ if (openRequest.roleId == roleId) {
+ results.add(openRequest);
+ openlist.remove();
+ count--;
+ }
+ }
+ return results;
+ }
+ /**
+ * Extract a specific number of placed requests for a role
+ * @param roleId role Id
+ * @param count count to extract
+ * @return a list of requests which are no longer in the placed request data structure
+ */
+ public synchronized List<OutstandingRequest> extractPlacedRequestsForRole(int roleId, int count) {
+ List<OutstandingRequest> results = new ArrayList<>();
+ Iterator<OutstandingRequest> iterator = placedRequests.keySet().iterator();
+ while (iterator.hasNext() && count > 0) {
+ OutstandingRequest request = iterator.next();
+ if (request.roleId == roleId) {
+ results.add(request);
+ count--;
+ }
+ }
+ // now cull them from the map
+ for (OutstandingRequest result : results) {
+ placedRequests.remove(result);
+ }
+
+ return results;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0b981b8..9ab40bd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -638,13 +638,13 @@ public class RoleHistory {
* @param actualCount current count of instances
* @return The allocation outcome
*/
- public synchronized ContainerAllocationOutcome onContainerAllocated(Container container,
+ public synchronized ContainerAllocation onContainerAllocated(Container container,
int desiredCount,
int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
- ContainerAllocationOutcome outcome =
+ ContainerAllocation outcome =
outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
@@ -874,4 +874,30 @@ public class RoleHistory {
long now = now();
return outstandingRequests.escalateOutstandingRequests(now);
}
+
+ /**
+ * Build the list of requests to cancel from the outstanding list.
+ * @param role
+ * @param toCancel
+ * @return a list of cancellable operations.
+ */
+ public synchronized List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
+ List<AbstractRMOperation> results = new ArrayList<>(toCancel);
+ // first scan through the unplaced request list to find all of a role
+ int roleId = role.getKey();
+ List<OutstandingRequest> requests =
+ outstandingRequests.extractOpenRequestsForRole(roleId, toCancel);
+
+ // are there any left?
+ int remaining = toCancel - requests.size();
+ // ask for some placed nodes
+ requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, remaining));
+
+ // build cancellations
+ for (OutstandingRequest request : requests) {
+ results.add(request.createCancelOperation());
+ }
+ return results;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index d962438..257092a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -25,6 +25,7 @@ import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerAssignment
import org.apache.slider.server.appmaster.state.RoleInstance
@@ -163,6 +164,23 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
}
}
-
+
+ @Test
+ public void testCancelWithRequestsOutstanding() throws Throwable {
+ // flex cluster size before the original set were allocated
+
+
+ role0Status.desired = 6
+ // build the ops
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ // here the data structures exist
+
+ // go down
+ role0Status.desired = 3
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.size() == 3
+ ops2.each { assert it instanceof CancelSingleRequest}
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index ee5eead..9ac6fcf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
@@ -27,10 +28,11 @@ import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.operations.CancelRequestOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.operations.RMOperationHandler
+import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerAssignment
import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
@@ -95,10 +97,17 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
assertListLength(ops, 5)
// now 5 outstanding requests.
assert role0.requested == 5
-
+
// allocate one
- role0.incActual()
- role0.decRequested()
+ List<AbstractRMOperation> processed = [ops[0]]
+ List<ContainerId> released = []
+ List<AppState.NodeCompletionResult> completionResults = []
+ submitOperations(processed, released)
+ List<RoleInstance> instances = createAndSubmitNodes(released)
+ processSubmissionOperations(instances, completionResults, released)
+
+
+ // four outstanding
assert role0.requested == 4
@@ -106,9 +115,10 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
role0.desired = 3
ops = appState.reviewRequestAndReleaseNodes()
- // expect a cancel operation from review
- assertListLength(ops, 1)
- assert ops[0] instanceof CancelRequestOperation
+ // expect two cancel operation from review
+ assertListLength(ops, 2)
+ ops.each { assert it instanceof CancelSingleRequest }
+
RMOperationHandler handler = new MockRMOperationHandler()
handler.availableToCancel = 4;
handler.execute(ops)
@@ -119,11 +129,10 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
role0.desired = 2
ops = appState.reviewRequestAndReleaseNodes()
assertListLength(ops, 1)
- assert ops[0] instanceof CancelRequestOperation
+ ops.each { assert it instanceof CancelSingleRequest }
handler.execute(ops)
assert handler.availableToCancel == 1
assert role0.requested == 1
-
}
@Test
@@ -136,9 +145,8 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
assert role0.requested == 5
role0.desired = 0
ops = appState.reviewRequestAndReleaseNodes()
- assertListLength(ops, 1)
- CancelRequestOperation cancel = ops[0] as CancelRequestOperation
- assert cancel.count == 5
+ assertListLength(ops, 5)
+
}
@@ -160,7 +168,9 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
role0.desired = 1;
assert role0.delta == -3
ops = appState.reviewRequestAndReleaseNodes()
- assertListLength(ops, 2)
+ assertListLength(ops, 3)
+ assert 2 == (ops.findAll {it instanceof CancelSingleRequest}).size()
+ assert 1 == (ops.findAll {it instanceof ContainerReleaseOperation}).size()
assert role0.requested == 0
assert role0.releasing == 1
}
@@ -171,31 +181,31 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
// role: desired = 2, requested = 1, actual=1
def role0 = role0Status
role0.desired = 2
- role0.incRequested()
- role0.incRequested()
List<AbstractRMOperation> ops
-
- // there are now two outstanding, two actual
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert 2 == (ops.findAll { it instanceof ContainerRequestOperation }).size()
+
+ // there are now two outstanding, two actual
// Release 3 and verify that the two
// cancellations were combined with a release
role0.desired = 0;
ops = appState.reviewRequestAndReleaseNodes()
- assertListLength(ops, 1)
- CancelRequestOperation cancel = ops[0] as CancelRequestOperation
- assert cancel.getCount() == 2
+ assert ops.size() == 2
+ assert 2 == (ops.findAll { it instanceof CancelSingleRequest }).size()
}
@Test
public void testFlexUpOutstandingRequests() throws Throwable {
-
- // role: desired = 2, requested = 1, actual=1
+
+ List<AbstractRMOperation> ops
+ // role: desired = 2, requested = 1, actual=1
def role0 = role0Status
role0.desired = 2
role0.incActual();
role0.incRequested()
-
- List<AbstractRMOperation> ops
+
+
// flex up 2 nodes, yet expect only one node to be requested,
// as the outstanding request is taken into account
@@ -283,7 +293,9 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> releases = []
appState.onContainersAllocated(allocations, assignments, releases)
- assertListLength(releases, 0)
+ // we expect four release requests here for all the allocated containers
+ assertListLength(releases, 4)
+ releases.each { assert it instanceof CancelSingleRequest }
assertListLength(assignments, 4)
assignments.each { ContainerAssignment assigned ->
Container target = assigned.container
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index 8fd9858..4726e71 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.ContainerAssignment
@@ -59,12 +60,19 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest
assert request.relaxLocality
assert request.nodes == null
assert request.racks == null
+ assert request.capability
Container allocated = engine.allocateContainer(request)
List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated([(Container)allocated], assignments, operations)
- assert operations.size() == 0
+ List<AbstractRMOperation> releaseOperations = []
+ appState.onContainersAllocated([(Container)allocated], assignments, releaseOperations)
+ // verify the release matches the allocation
+ assert releaseOperations.size() == 1
+ CancelSingleRequest cancelOp = releaseOperations[0] as CancelSingleRequest;
+ assert cancelOp.request
+ assert cancelOp.request.capability
+ assert cancelOp.request.capability.equals(allocated.resource)
+ // now the assignment
assert assignments.size() == 1
ContainerAssignment assigned = assignments[0]
Container container = assigned.container
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 3f41dfd..3d396f8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -23,8 +23,6 @@ import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockContainer
-import org.apache.slider.server.appmaster.model.mock.MockNodeId
import org.apache.slider.server.appmaster.model.mock.MockPriority
import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -63,7 +61,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
tracker.newRequest(host1, 0)
tracker.newRequest(host2, 0)
tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1", null) == ContainerAllocationOutcome.Placed
+ assert tracker.onContainerAllocated(1, "host1", null).outcome == ContainerAllocationOutcome.Placed
assert !tracker.lookupPlacedRequest(1, "host1")
assert tracker.lookupPlacedRequest(0, "host1")
}
@@ -91,7 +89,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
resource.virtualCores=1
resource.memory = 48;
c1.setResource(resource)
- ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1)
+ ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1).outcome
assert outcome == ContainerAllocationOutcome.Unallocated
assert tracker.listOpenRequests().size() == 1
}
@@ -120,9 +118,11 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert issued.capability == resource
assert issued.priority.priority == c1.getPriority().getPriority()
assert req1.resourceRequirementsMatch(resource)
- ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, nodeId.host, c1)
+
+ def allocation = tracker.onContainerAllocated(0, nodeId.host, c1)
assert tracker.listOpenRequests().size() == 0
- assert outcome == ContainerAllocationOutcome.Open
+ assert allocation.outcome == ContainerAllocationOutcome.Open
+ assert allocation.origin.is(req1)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 34a0a4d..82750a3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -143,11 +143,11 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
}
public void assertOnContainerAllocated(Container c1, int p1, int p2) {
- assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2)
+ assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2).outcome
}
public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
- assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2)
+ assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2).outcome
}
def assertNoOutstandingPlacedRequests() {
@@ -190,7 +190,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
- assert ContainerAllocationOutcome.Unallocated == roleHistory.onContainerAllocated(container, 3, 3)
+ assert ContainerAllocationOutcome.Unallocated ==
+ roleHistory.onContainerAllocated(container, 3, 3).outcome
// yet the list is now empty
assertNoOutstandingPlacedRequests()
roleHistory.listOpenRequests().empty
@@ -198,7 +199,6 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
assertListEquals([age2Active0], a2)
-
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
index a027098..ca5d805 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
@@ -114,6 +114,7 @@ class Allocator {
container.nodeId = node.nodeId
container.nodeHttpAddress = node.httpAddress()
container.priority = request.priority
+ container.resource = request.capability
return container;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b952b640/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 04466c6..e3d509a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -114,8 +114,8 @@ class MockYarnEngine {
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
Container container = allocateContainer(req.request)
- log.info("allocated container $container for $req")
if (container != null) {
+ log.info("allocated container $container for $req")
allocation.add(container)
} else {
log.debug("Unsatisfied allocation $req")
[04/20] incubator-slider git commit: Merge branch 'develop' into
feature/SLIDER-799-AM-managed-relax
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-799-AM-managed-relax
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7141d931
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7141d931
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7141d931
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 7141d931b3a48111f97ad9bbff0018e087ff2979
Parents: b2b8750 909b7c1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 3 16:01:49 2015 -0800
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 3 16:01:49 2015 -0800
----------------------------------------------------------------------
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 2 +-
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 2 +-
app-packages/storm/package/scripts/params.py | 1 +
.../storm/package/templates/config.yaml.j2 | 2 +-
.../slider/api/proto/RestTypeMarshalling.java | 14 ++-
.../org/apache/slider/client/SliderClient.java | 27 +++-
.../client/ipc/SliderClusterOperations.java | 8 +-
.../slider/common/params/ActionListArgs.java | 4 +
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/tools/SliderUtils.java | 22 +++-
.../providers/agent/AgentProviderService.java | 24 ++--
.../services/security/CertificateManager.java | 19 +--
.../services/security/KeystoreGenerator.java | 8 +-
.../server/services/security/SecurityStore.java | 66 ++++++++++
.../security/SecurityStoreGenerator.java | 8 +-
.../services/security/StoresGenerator.java | 21 ++--
.../services/security/TruststoreGenerator.java | 9 +-
.../security/TestCertificateManager.java | 123 +++++++++++++------
18 files changed, 274 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
[16/20] incubator-slider git commit: SLIDER-799 tests are all working
Posted by st...@apache.org.
SLIDER-799 tests are all working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad41b244
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad41b244
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad41b244
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: ad41b2444c68a27ab9a5d1a10470da69f74f7bb6
Parents: 3dd2f72
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 19 17:07:50 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 19 17:07:50 2015 +0000
----------------------------------------------------------------------
.../slider/providers/PlacementPolicy.java | 10 +-
.../slider/server/appmaster/state/AppState.java | 4 +-
.../appmaster/state/OutstandingRequest.java | 120 +++++++++++++------
.../state/OutstandingRequestTracker.java | 2 +-
.../server/appmaster/state/RoleHistory.java | 3 +-
.../server/appmaster/state/RoleStatus.java | 22 ++--
.../TestMockAppStateDynamicRoles.groovy | 3 +-
.../TestMockAppStateRolePlacement.groovy | 2 +-
...tRoleHistoryOutstandingRequestTracker.groovy | 109 ++++++++++++-----
.../TestRoleHistoryRequestTracking.groovy | 28 ++---
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
.../appmaster/model/mock/MockFactory.groovy | 13 +-
12 files changed, 211 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index dc6c910..4e85a93 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -19,12 +19,13 @@
package org.apache.slider.providers;
/**
- * Placement values
+ * Placement values.
+ * This is nominally a bitmask, though not all values make sense
*/
public class PlacementPolicy {
/**
- * Default values
+ * Default value: history used, anti-affinity hinted at on rebuild/flex up
*/
public static final int DEFAULT = 0;
@@ -35,11 +36,12 @@ public class PlacementPolicy {
public static final int STRICT = 1;
/**
- * No data locality; do not bother trying to ask for any location
+ * No data locality; do not use placement history
*/
public static final int NO_DATA_LOCALITY = 2;
+
/**
- * Anti-affinity is mandatory.
+ * Anti-affinity is mandatory. This is not supported in YARN
*/
public static final int ANTI_AFFINITY_REQUIRED = 4;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index b7ca526..0f07ee9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -766,7 +766,7 @@ public class AppState {
// Add all the existing roles
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- if (roleStatus.getExcludeFromFlexing()) {
+ if (roleStatus.isExcludeFromFlexing()) {
// skip inflexible roles, e.g AM itself
continue;
}
@@ -1792,7 +1792,7 @@ public class AppState {
log.debug("in reviewRequestAndReleaseNodes()");
List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- if (!roleStatus.getExcludeFromFlexing()) {
+ if (!roleStatus.isExcludeFromFlexing()) {
List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
allOperations.addAll(operations);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 4fd2933..8c320f0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -64,27 +64,32 @@ public final class OutstandingRequest {
* <p>
* Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
*/
- public AMRMClient.ContainerRequest issuedRequest;
+ private AMRMClient.ContainerRequest issuedRequest;
/**
* Requested time in millis.
* <p>
* Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
*/
- public long requestedTimeMillis;
+ private long requestedTimeMillis;
/**
* Time in millis after which escalation should be triggered..
* <p>
* Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
*/
- public long escalationTimeoutMillis;
+ private long escalationTimeoutMillis;
/**
* Has the placement request been escalated?
*/
- public boolean escalated;
-
+ private boolean escalated;
+
+ /**
+ * Flag to indicate that escalation is allowed
+ */
+ private boolean mayEscalate;
+
/**
* Create a request
* @param roleId role
@@ -98,7 +103,7 @@ public final class OutstandingRequest {
}
/**
- * Create an outsanding request with the given role and hostname
+ * Create an outstanding request with the given role and hostname
* Important: this is useful only for map lookups -the other constructor
* with the NodeInstance parameter is needed to generate node-specific
* container requests
@@ -113,14 +118,40 @@ public final class OutstandingRequest {
/**
* Is the request located in the cluster, that is: does it have a node.
- * @return
+ * @return true if a node instance was supplied in the constructor
*/
public boolean isLocated() {
return node != null;
}
-
+
+ public long getRequestedTimeMillis() {
+ return requestedTimeMillis;
+ }
+
+ public long getEscalationTimeoutMillis() {
+ return escalationTimeoutMillis;
+ }
+
+ public boolean isEscalated() {
+ return escalated;
+ }
+
+ public boolean mayEscalate() {
+ return mayEscalate;
+ }
+
+ public AMRMClient.ContainerRequest getIssuedRequest() {
+ return issuedRequest;
+ }
+
/**
* Build a container request.
+ * <p>
+ * The value of {@link #node} is used to direct a lot of policy. If null,
+ * placement is relaxed.
+ * If not null, the choice of whether to use the suggested node
+ * is based on the placement policy and failure history.
+ * <p>
* If the request has an address, it is set in the container request
* (with a flag to enable relaxed priorities).
* <p>
@@ -134,37 +165,54 @@ public final class OutstandingRequest {
*/
public synchronized AMRMClient.ContainerRequest buildContainerRequest(
Resource resource, RoleStatus role, long time, String labelExpression) {
- String[] hosts;
- boolean relaxLocality;
+ Preconditions.checkArgument(resource != null, "null `resource` arg");
+ Preconditions.checkArgument(role != null, "null `role` arg");
+
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
- boolean usePlacementHistory = role.isStrictOrAntiAffinePlacement();
- if (!usePlacementHistory) {
- // If strict placement does not mandate using placement then check
- // that the recent failures on this node is not higher than threshold
- if (node != null) {
- int numFailuresOnLastHost = node.get(role.getKey()).getFailedRecently();
- usePlacementHistory = numFailuresOnLastHost <= role.getNodeFailureThreshold();
- if(!usePlacementHistory) {
- log.info("Recent node failures {} is higher than threshold {}. Dropping host {} from preference.",
- numFailuresOnLastHost, role.getNodeFailureThreshold(), node.hostname);
- }
+ String[] hosts;
+ boolean relaxLocality;
+ boolean strictPlacement = role.isStrictPlacement();
+ NodeInstance target = this.node;
+ if (target != null) {
+ // there is a host specified; get its details
+
+ // tell the node it is in play
+ NodeEntry entry = target.getOrCreate(roleId);
+ // failure count
+ int numFailuresOnLastHost = entry != null ? entry.getFailedRecently() : 0;
+
+ // which on non-strict placement may have some effect
+ if (!strictPlacement && numFailuresOnLastHost > role.getNodeFailureThreshold()) {
+ // too many failures for this node
+ log.info("Recent node failures {} is higher than threshold {}. Not requesting host {}",
+ numFailuresOnLastHost, role.getNodeFailureThreshold(), target.hostname);
+ // reset the target node so this request is downgraded
+ target = null;
}
}
- if (node != null && usePlacementHistory) {
+ if (target != null) {
+ // placed request. Hostname is used in request
hosts = new String[1];
- hosts[0] = node.hostname;
- relaxLocality = !role.isStrictOrAntiAffinePlacement();
- // tell the node it is in play
- node.getOrCreate(roleId);
+ hosts[0] = target.hostname;
+ // and locality flag is set to false; Slider will decide when
+ // to relax things
+ relaxLocality = false;
+
log.info("Submitting request for container on {}", hosts[0]);
+ // enable escalation for all but strict placements.
+ mayEscalate = !strictPlacement;
escalated = false;
} else {
- // the placement is implicitly escalated.
- escalated = true;
+ // no hosts
hosts = null;
+ // relax locality is mandatory on an unconstrained placement
relaxLocality = true;
+ // declare that the the placement is implicitly escalated.
+ escalated = true;
+ // and forbid it happening
+ mayEscalate = false;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
issuedRequest = new AMRMClient.ContainerRequest(resource,
@@ -197,7 +245,6 @@ public final class OutstandingRequest {
nodes = null;
}
-
AMRMClient.ContainerRequest newRequest =
new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
nodes,
@@ -218,14 +265,17 @@ public final class OutstandingRequest {
}
/**
- * Query to see if the request is ready to be escalated
+ * Query to see if the request is available and ready to be escalated
* @param time time to check against
* @return true if escalation should begin
*/
public synchronized boolean shouldEscalate(long time) {
- return issuedRequest != null && !escalated && escalationTimeoutMillis < time;
+ return mayEscalate
+ && !escalated
+ && issuedRequest != null
+ && escalationTimeoutMillis < time;
}
-
+
/**
* Equality is on hostname and role
* @param o other
@@ -265,14 +315,16 @@ public final class OutstandingRequest {
}
@Override
- public synchronized String toString() {
+ public String toString() {
final StringBuilder sb = new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
sb.append(", node=").append(node);
sb.append(", hostname='").append(hostname).append('\'');
+ sb.append(", issuedRequest=").append(issuedRequest);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
- sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
+ sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
+ sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 48f6e57..959cb1f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -251,7 +251,7 @@ public class OutstandingRequestTracker {
if (outstandingRequest.shouldEscalate(now)) {
// time to escalate
- CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.issuedRequest);
+ CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.getIssuedRequest());
operations.add(cancel);
AMRMClient.ContainerRequest escalated =
outstandingRequest.escalate();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 6aefc47..a1b54c7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -481,7 +481,8 @@ public class RoleHistory {
*/
@VisibleForTesting
public synchronized NodeInstance findNodeForNewInstance(RoleStatus role) {
- if (role.getNoDataLocality()) {
+ if (!role.isPlacementDesired()) {
+ // no data locality policy
return null;
}
int roleKey = role.getKey();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 4ee8fad..899948f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -87,24 +87,24 @@ public final class RoleStatus implements Cloneable {
return providerRole.nodeFailureThreshold;
}
- public boolean getExcludeFromFlexing() {
- return 0 != (getPlacementPolicy() & PlacementPolicy.EXCLUDE_FROM_FLEXING);
+ public boolean isExcludeFromFlexing() {
+ return hasPlacementPolicy(PlacementPolicy.EXCLUDE_FROM_FLEXING);
}
-
- public boolean getNoDataLocality() {
- return 0 != (getPlacementPolicy() & PlacementPolicy.NO_DATA_LOCALITY);
- }
-
+
public boolean isStrictPlacement() {
- return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT);
+ return hasPlacementPolicy(PlacementPolicy.STRICT);
}
public boolean isAntiAffinePlacement() {
- return 0 != (getPlacementPolicy() & PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+ return hasPlacementPolicy(PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+ }
+
+ public boolean hasPlacementPolicy(int policy) {
+ return 0 != (getPlacementPolicy() & policy);
}
- public boolean isStrictOrAntiAffinePlacement() {
- return isStrictPlacement() || isAntiAffinePlacement();
+ public boolean isPlacementDesired() {
+ return !hasPlacementPolicy(PlacementPolicy.NO_DATA_LOCALITY);
}
public synchronized int getDesired() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 13ecf13..ee4abd6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -168,7 +168,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
assert instances.size() == 1
def instanceA = instances.find { RoleInstance instance ->
- instance.roleId = ID4
+ instance.roleId == ID4
}
assert instanceA
def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
@@ -193,7 +193,6 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
def actions = appState.reviewRequestAndReleaseNodes()
assert actions.size() == 1
- assertRelaxLocalityFlag(ID4, "", true, actions)
ContainerRequestOperation cro = (ContainerRequestOperation) actions[0]
def nodes = cro.request.nodes
assert nodes.size() == 1
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index e9de390..8fd9858 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -101,7 +101,7 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest
AMRMClient.ContainerRequest request2 = operation.request
assert request2 != null
assert request2.nodes[0] == containerHostname
- assert request2.relaxLocality
+ assert !request2.relaxLocality
engine.execute(ops)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 2fe6763..97d970d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.model.history
import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -26,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -64,9 +66,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testResetEntries() throws Throwable {
- OutstandingRequest r1 = tracker.newRequest(host1, 0)
- OutstandingRequest r2 = tracker.newRequest(host2, 0)
- OutstandingRequest r3 = tracker.newRequest(host1, 1)
+ tracker.newRequest(host1, 0)
+ tracker.newRequest(host2, 0)
+ tracker.newRequest(host1, 1)
List<NodeInstance> canceled = tracker.resetOutstandingRequests(0)
assert canceled.size() == 2
assert canceled.contains(host1)
@@ -78,26 +80,17 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert tracker.resetOutstandingRequests(1).size() == 1
}
-/*
- @Override
- AggregateConf buildInstanceDefinition() {
- def aggregateConf = super.buildInstanceDefinition()
- def component0 = aggregateConf.resourceOperations.getMandatoryComponent(ROLE0)
- component0.set(ResourceKeys.COMPONENT_PLACEMENT_POLICY, PlacementPolicy.STRICT)
- return aggregateConf
- }
-*/
@Test
public void testEscalation() throws Throwable {
- ProviderRole providerRole1 = role1Status.providerRole
- assert providerRole1.placementPolicy == PlacementPolicy.STRICT;
- // first request
- final def (res1, outstanding1) = newRequest(role1Status)
- final def initialRequest = outstanding1.buildContainerRequest(res1, role1Status, 0, null)
- assert outstanding1.issuedRequest != null;
- assert outstanding1.located
- assert !outstanding1.escalated
+
+ // first request: default placement
+ assert role0Status.placementPolicy == PlacementPolicy.DEFAULT;
+ final def (res0, outstanding0) = newRequest(role0Status)
+ final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ assert outstanding0.issuedRequest != null;
+ assert outstanding0.located
+ assert !outstanding0.escalated
assert !initialRequest.relaxLocality
assert tracker.listOutstandingRequests().size() == 1
@@ -109,32 +102,86 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
def (res2, outstanding2) = newRequest(role2Status)
// simulate some time escalation of role 1 MUST now be triggered
- List<AbstractRMOperation> escalations =
- tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000 + 500 )
+ final def interval = role0Status.placementTimeoutSeconds * 1000 + 500
+ def now = interval
+ final List<AbstractRMOperation> escalations = tracker.escalateOutstandingRequests(now)
- assert outstanding1.escalated
+ assert outstanding0.escalated
assert !outstanding2.escalated
// two entries
assert escalations.size() == 2;
final def e1 = escalations[0]
assert e1 instanceof CancelSingleRequest
- CancelSingleRequest cancel = (CancelSingleRequest) e1
+ final CancelSingleRequest cancel = (CancelSingleRequest) e1
assert initialRequest == cancel.request
final def e2 = escalations[1]
assert e2 instanceof ContainerRequestOperation;
final def escRequest = (ContainerRequestOperation) e2
- def req2 = escRequest.request
- assert req2.relaxLocality
+ assert escRequest.request.relaxLocality
+
+ // build that second request from an anti-affine entry
+ // these get placed as well
+ now += interval
+ final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now, null)
+ // escalate a little bit more
+ final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now)
+ // and expect no new entries
+ assert escalations2.size() == 0
- def (res3, outstanding3) = newRequest(role2Status)
- outstanding3.buildContainerRequest(res3, role2Status, 0, null)
+ // go past the role2 timeout
+ now += role2Status.placementTimeoutSeconds * 1000 + 500
+ // escalate a little bit more
+ final List<AbstractRMOperation> escalations3 = tracker.escalateOutstandingRequests(now)
+ // and expect another escalation
+ assert escalations3.size() == 2
+ assert outstanding2.escalated
+
+ // finally add a strict entry to th emix
+ def (res3, outstanding3) = newRequest(role1Status)
+ final ProviderRole providerRole1 = role1Status.providerRole
+ assert providerRole1.placementPolicy == PlacementPolicy.STRICT
+ now += interval
+ assert !outstanding3.mayEscalate
+ final List<AbstractRMOperation> escalations4 = tracker.escalateOutstandingRequests(now)
+ assert escalations4.empty
- List<AbstractRMOperation> escalations2 =
- tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000 + 500)
- assert escalations2.size() == 0
}
+ @Test
+ public void testPlacementSkipsFailures() throws Throwable {
+ final def (res0, outstanding0) = newRequest(role0Status)
+ def entry = host1.getOrCreate(role0Status.key)
+ entry.containerCompleted(false)
+ entry.containerCompleted(false)
+ entry.containerCompleted(false)
+ assert entry.failedRecently == 3
+ final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ assert initialRequest.relaxLocality
+ assert initialRequest.nodes == null
+ }
+
+ @Test
+ public void testStrictPlacementDoesntSkipFailures() throws Throwable {
+ def roleStatus = role1Status
+ assert roleStatus.strictPlacement
+ final def (res0, outstanding0) = newRequest(roleStatus)
+ def entry = host1.getOrCreate(roleStatus.key)
+ entry.containerCompleted(false)
+ entry.containerCompleted(false)
+ entry.containerCompleted(false)
+ assert entry.failedRecently == 3
+ final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0,
+ roleStatus, 0, null)
+ assert !initialRequest.relaxLocality
+ assert initialRequest.nodes[0] == host1.hostname
+ }
+
+ /**
+ * Create a new request (always against host1)
+ * @param r
+ * @return
+ */
public def newRequest(RoleStatus r) {
final Resource res2 = new MockResource()
appState.buildResourceRequirements(r, res2)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index e593eab..e84dfce 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -122,7 +122,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
@@ -130,28 +130,28 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.listOutstandingPlacedRequests.empty
+ assert roleHistory.listOutstandingPlacedRequests().empty
}
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.listOutstandingPlacedRequests.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests().size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 2, 2)
- assert roleHistory.listOutstandingPlacedRequests.empty
+ assert roleHistory.listOutstandingPlacedRequests().empty
}
@@ -160,22 +160,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.listOutstandingPlacedRequests.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests().size() == 1
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.listOutstandingPlacedRequests.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests().size() == 1
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
assert !roleHistory.onContainerAllocated(container, 3, 3)
// yet the list is now empty
- assert roleHistory.listOutstandingPlacedRequests.empty
+ assert roleHistory.listOutstandingPlacedRequests().empty
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -189,17 +189,17 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- assert roleHistory.listOutstandingPlacedRequests.size() == 2
+ assert roleHistory.listOutstandingPlacedRequests().size() == 2
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 3, 1)
- assert roleHistory.listOutstandingPlacedRequests.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests().size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.listOutstandingPlacedRequests.empty
+ assert roleHistory.listOutstandingPlacedRequests().empty
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 3)
- assert roleHistory.listOutstandingPlacedRequests.empty
+ assert roleHistory.listOutstandingPlacedRequests().empty
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 6c83c55..f30fce6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -135,7 +135,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
/**
* Build a role instance from a container assignment
* @param assigned
- * @return
+ * @return the instance
*/
RoleInstance roleInstance(ContainerAssignment assigned) {
Container target = assigned.container
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 37f4021..06bc10c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -40,18 +40,23 @@ class MockFactory implements MockRoles {
public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
MockRoles.ROLE0,
- 0)
+ 0,
+ PlacementPolicy.DEFAULT,
+ 2,
+ 1)
+ // role 1 is strict. timeout should be irrelevant; same as failures
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
1,
PlacementPolicy.STRICT,
- 3,
+ 2,
1)
+ // role 2: longer delay
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
PlacementPolicy.ANTI_AFFINITY_REQUIRED,
- 4,
+ 2,
2)
int appIdCount;
int attemptIdCount;
@@ -180,8 +185,6 @@ class MockFactory implements MockRoles {
def roleMap(int count) {
return [
(ResourceKeys.COMPONENT_INSTANCES):count.toString(),
- (ResourceKeys.COMPONENT_PLACEMENT_POLICY):"${PlacementPolicy.STRICT}".toString()
-
]
}
[12/20] incubator-slider git commit: SLIDER-799 quick code review of
this while looking @ log aggregation setup
Posted by st...@apache.org.
SLIDER-799 quick code review of this while looking @ log aggregation setup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/06ac342f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/06ac342f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/06ac342f
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 06ac342fc86da22f3fbbec8fbaabd2c57c77ace7
Parents: 1624bdf
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 10:51:22 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 10:51:22 2015 +0000
----------------------------------------------------------------------
.../slider/core/launch/AbstractLauncher.java | 19 +++++++++----------
.../slider/core/launch/AppMasterLauncher.java | 6 +++---
.../slider/core/launch/ClasspathConstructor.java | 2 +-
.../slider/core/launch/CommandLineBuilder.java | 2 +-
4 files changed, 14 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ac342f/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 3051c0b..3b12877 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -57,6 +57,7 @@ public abstract class AbstractLauncher extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
public static final String CLASSPATH = "CLASSPATH";
+ public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
/**
* Filesystem to use for the launch
*/
@@ -64,15 +65,13 @@ public abstract class AbstractLauncher extends Configured {
/**
* Env vars; set up at final launch stage
*/
- protected final Map<String, String> envVars = new HashMap<String, String>();
+ protected final Map<String, String> envVars = new HashMap<>();
protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<String>(20);
- protected final Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
- private final Map<String, ByteBuffer> serviceData =
- new HashMap<String, ByteBuffer>();
+ protected final List<String> commands = new ArrayList<>(20);
+ protected final Map<String, LocalResource> localResources = new HashMap<>();
+ private final Map<String, ByteBuffer> serviceData = new HashMap<>();
// security
protected final Credentials credentials = new Credentials();
protected LogAggregationContext logAggregationContext;
@@ -84,7 +83,7 @@ public abstract class AbstractLauncher extends Configured {
this.coreFileSystem = fs;
}
- public AbstractLauncher(CoreFileSystem fs) {
+ protected AbstractLauncher(CoreFileSystem fs) {
this.coreFileSystem = fs;
}
@@ -212,7 +211,7 @@ public abstract class AbstractLauncher extends Configured {
log.debug("{} tokens", credentials.numberOfTokens());
DataOutputBuffer dob = new DataOutputBuffer();
String tokenFileName =
- this.getConf().get("mapreduce.job.credentials.binary");
+ this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY);
if (tokenFileName != null) {
// use delegation tokens, i.e. from Oozie
Credentials creds =
@@ -285,7 +284,7 @@ public abstract class AbstractLauncher extends Configured {
String logPatternJoinStr = "|";
MapOperations options = new MapOperations("", map);
- List<String> logIncludePatterns = new ArrayList<String>();
+ List<String> logIncludePatterns = new ArrayList<>();
String includePatternExpression = options.getOption(
ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
if (!includePatternExpression.isEmpty()) {
@@ -302,7 +301,7 @@ public abstract class AbstractLauncher extends Configured {
logPatternJoinStr);
log.info("Log include patterns: {}", logIncludePattern);
- List<String> logExcludePatterns = new ArrayList<String>();
+ List<String> logExcludePatterns = new ArrayList<>();
String excludePatternExpression = options.getOption(
ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
if (!excludePatternExpression.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ac342f/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index 740badd..b4ca791 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -221,7 +221,7 @@ public class AppMasterLauncher extends AbstractLauncher {
Token<? extends TokenIdentifier>[] tokens = null;
boolean tokensProvided =
- this.getConf().get("mapreduce.job.credentials.binary") != null;
+ this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
if (!tokensProvided) {
// For now, only getting tokens for the default file-system.
FileSystem fs = coreFileSystem.getFileSystem();
@@ -239,8 +239,8 @@ public class AppMasterLauncher extends AbstractLauncher {
if (!tokensProvided) {
log.warn("No HDFS delegation tokens obtained for AM launch context");
} else {
- log.info("Tokens provided via mapreduce.job.credentials.binary property "
- + "being leveraged for AM launch");
+ log.info("Tokens provided via "+ MAPREDUCE_JOB_CREDENTIALS_BINARY +" property "
+ + "being used for AM launch");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ac342f/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index f474ffd..3ba0181 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -39,7 +39,7 @@ public class ClasspathConstructor {
public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
- private final List<String> pathElements = new ArrayList<String>();
+ private final List<String> pathElements = new ArrayList<>();
public ClasspathConstructor() {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ac342f/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index f50bb48..dbaa981 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -30,7 +30,7 @@ import java.util.List;
* Special support for JVM command buildup.
*/
public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<String>(20);
+ protected final List<String> argumentList = new ArrayList<>(20);
/**
[03/20] incubator-slider git commit: SLIDER-799 AM-managed-placement.
Starting to implement this; saving current work
Posted by st...@apache.org.
SLIDER-799 AM-managed-placement. Starting to implement this; saving current work
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2b8750e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2b8750e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2b8750e
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: b2b8750ee28688b4b07055086d68ff8edf043f2b
Parents: 8c8682f
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 3 16:01:10 2015 -0800
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 3 16:01:10 2015 -0800
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 18 ++++
.../apache/slider/providers/ProviderRole.java | 24 ++++-
.../slideram/SliderAMClientProvider.java | 3 +-
.../slider/server/appmaster/state/AppState.java | 15 ++--
.../appmaster/state/OutstandingRequest.java | 7 +-
.../state/OutstandingRequestTracker.java | 93 ++++++++++++++------
.../server/appmaster/state/RoleHistory.java | 4 +-
7 files changed, 125 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 50ca82f..ce2a54f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -140,6 +140,24 @@ public interface ResourceKeys {
*/
int DEFAULT_NODE_FAILURE_THRESHOLD = 3;
+
+ /**
+ * Time in seconds to relax placement delay
+ */
+ String PLACEMENT_RELAX_DELAY =
+ "yarn.placement.relax.delay.seconds";
+
+ /**
+ * Time to have a strict placement policy outstanding before
+ * downgrading to a lax placement (for those components which permit that).
+ * <ol>
+ * <li>For strictly placed components, there's no relaxation.</li>
+ * <li>For components with no locality, there's no need to relax</li>
+ * </ol>
+ *
+ */
+ int DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS = 30;
+
/**
* Log aggregation include, exclude patterns
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 17124d2..7caae48 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -31,16 +31,36 @@ public final class ProviderRole {
public final int id;
public final int placementPolicy;
public final int nodeFailureThreshold;
+ public final long placementTimeoutSeconds;
public ProviderRole(String name, int id) {
- this(name, id, PlacementPolicy.DEFAULT, ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD);
+ this(name,
+ id,
+ PlacementPolicy.DEFAULT,
+ ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+ ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS);
}
- public ProviderRole(String name, int id, int policy, int nodeFailureThreshold) {
+ /**
+ * Create a provider role
+ * @param name role/component name
+ * @param id ID. This becomes the YARN priority
+ * @param policy placement policy
+ * @param nodeFailureThreshold threshold for node failures (within a reset interval)
+ * after which a node failure is considered an app failure
+ * @param placementTimeoutSeconds for lax placement, timeout in seconds before
+ * a relaxed placement request is generated.
+ */
+ public ProviderRole(String name,
+ int id,
+ int policy,
+ int nodeFailureThreshold,
+ long placementTimeoutSeconds) {
this.name = name;
this.id = id;
this.placementPolicy = policy;
this.nodeFailureThreshold = nodeFailureThreshold;
+ this.placementTimeoutSeconds = placementTimeoutSeconds;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 1666c84..abb9648 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -89,7 +89,8 @@ public class SliderAMClientProvider extends AbstractClientProvider
public static final ProviderRole APPMASTER =
new ProviderRole(COMPONENT_AM, KEY_AM,
PlacementPolicy.EXCLUDE_FROM_FLEXING,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD);
+ ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+ 0);
/**
* Initialize role list
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9e2d2cf..4713ef1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -630,7 +630,6 @@ public class AppState {
* @return a new provider role
* @throws BadConfigException bad configuration
*/
- @VisibleForTesting
public ProviderRole createDynamicProviderRole(String name,
MapOperations component) throws
BadConfigException {
@@ -639,13 +638,19 @@ public class AppState {
ResourceKeys.COMPONENT_PRIORITY,
priOpt, 0, 1, -1);
String placementOpt = component.getOption(
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.DEFAULT));
int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
placementOpt, 0, 0, -1);
- ProviderRole newRole = new ProviderRole(name, priority, placement,
- getNodeFailureThresholdForRole(name));
+ int placementTimeout =
+ component.getOptionInt(ResourceKeys.PLACEMENT_RELAX_DELAY,
+ ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS);
+ ProviderRole newRole = new ProviderRole(name,
+ priority,
+ placement,
+ getNodeFailureThresholdForRole(name),
+ placementTimeout);
log.info("New {} ", newRole);
return newRole;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 2bc211a..3563a29 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -92,10 +92,13 @@ public final class OutstandingRequest {
/**
* Build a container request.
* If the request has an address, it is set in the container request
- * (with a flag to enable relaxed priorities)
+ * (with a flag to enable relaxed priorities).
+ * <p>
+ * This operation sets the requested time flag, used for tracking timeouts
+ * on outstanding requests
* @param resource resource
* @param role role
- * @param time: time to record
+ * @param time time to record as request time
* @param labelExpression label to satisfy
* @return the request to raise
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index d847962..f6dc2de 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -33,6 +33,7 @@ import java.util.Map;
/**
* Tracks outstanding requests made with a specific placement option.
+ * <p>
* If an allocation comes in that is not in the map: either the allocation
* was unplaced, or the placed allocation could not be met on the specified
* host, and the RM/scheduler fell back to another location.
@@ -42,13 +43,14 @@ public class OutstandingRequestTracker {
protected static final Logger log =
LoggerFactory.getLogger(OutstandingRequestTracker.class);
- private Map<OutstandingRequest, OutstandingRequest> requests =
- new HashMap<OutstandingRequest, OutstandingRequest>();
+ private Map<OutstandingRequest, OutstandingRequest> placedRequests =
+ new HashMap<>();
/**
* Create a new request for the specific role. If a
* location is set, the request is added to the list of requests to track.
- * if it isn't -it isn't.
+ * if it isn't, it is not tracked.
+ * <p>
* This does not update the node instance's role's request count
* @param instance node instance to manager
* @param role role index
@@ -58,7 +60,7 @@ public class OutstandingRequestTracker {
OutstandingRequest request =
new OutstandingRequest(role, instance);
if (request.isLocated()) {
- requests.put(request, request);
+ placedRequests.put(request, request);
}
return request;
}
@@ -70,7 +72,7 @@ public class OutstandingRequestTracker {
* @return the request or null if there was no outstanding one
*/
public synchronized OutstandingRequest lookup(int role, String hostname) {
- return requests.get(new OutstandingRequest(role, hostname));
+ return placedRequests.get(new OutstandingRequest(role, hostname));
}
/**
@@ -79,7 +81,7 @@ public class OutstandingRequestTracker {
* @return the request
*/
public synchronized OutstandingRequest remove(OutstandingRequest request) {
- return requests.remove(request);
+ return placedRequests.remove(request);
}
/**
@@ -91,7 +93,7 @@ public class OutstandingRequestTracker {
*/
public synchronized boolean onContainerAllocated(int role, String hostname) {
OutstandingRequest request =
- requests.remove(new OutstandingRequest(role, hostname));
+ placedRequests.remove(new OutstandingRequest(role, hostname));
if (request == null) {
return false;
} else {
@@ -100,14 +102,45 @@ public class OutstandingRequestTracker {
}
return true;
}
-
+
+ /**
+ * Determine which host was a role type most recently used on, so that
+ * if a choice is made of which (potentially surplus) containers to use,
+ * the most recent one is picked first. This operation <i>does not</i>
+ * change the role history, though it queries it.
+ */
static class newerThan implements Comparator<Container>, Serializable {
private RoleHistory rh;
public newerThan(RoleHistory rh) {
this.rh = rh;
}
-
+
+ /**
+ * Get the age of a container. If it is not known in the history,
+ * return 0.
+ * @param c container
+ * @return age, null if
+ */
+ private long getAgeOf(Container c) {
+ long age = 0;
+ NodeInstance node = rh.getExistingNodeInstance(c);
+ int role = ContainerPriority.extractRole(c);
+ if (node != null) {
+ NodeEntry nodeEntry = node.get(role);
+ if (nodeEntry != null) {
+ age = nodeEntry.getLastUsed();
+ }
+ }
+ return age;
+ }
+
+ /**
+ * Comparator: which host is more recent?
+ * @param c1 container 1
+ * @param c2 container 2
+ * @return 1 if c2 older-than c1, 0 if equal; -1 if c1 older-than c2
+ */
@Override
public int compare(Container c1, Container c2) {
int role1 = ContainerPriority.extractRole(c1);
@@ -115,9 +148,8 @@ public class OutstandingRequestTracker {
if (role1 < role2) return -1;
if (role1 > role2) return 1;
- NodeInstance o1 = rh.getOrCreateNodeInstance(c1), o2 = rh.getOrCreateNodeInstance(c2);
- long age = o1.getOrCreate(role1).getLastUsed();
- long age2 = o2.getOrCreate(role1).getLastUsed();
+ long age = getAgeOf(c1);
+ long age2 = getAgeOf(c2);
if (age > age2) {
return -1;
@@ -128,26 +160,28 @@ public class OutstandingRequestTracker {
return 0;
}
}
+
/**
* Take a list of requests and split them into specific host requests and
* generic assignments. This is to give requested hosts priority
* in container assignments if more come back than expected
* @param rh RoleHistory instance
- * @param allocatedContainers the list of allocated containers
- * @param requested empty list of requested locations
- * @param unrequested empty list of unrequested hosts
+ * @param inAllocated the list of allocated containers
+ * @param outRequested initially empty list of requested locations
+ * @param outUnrequested initially empty list of unrequested hosts
*/
- public synchronized void partitionRequests(RoleHistory rh, List<Container> allocatedContainers,
- List<Container> requested,
- List<Container> unrequested) {
- Collections.sort(allocatedContainers, new newerThan(rh));
- for (Container container : allocatedContainers) {
+ public synchronized void partitionRequests(RoleHistory rh,
+ List<Container> inAllocated,
+ List<Container> outRequested,
+ List<Container> outUnrequested) {
+ Collections.sort(inAllocated, new newerThan(rh));
+ for (Container container : inAllocated) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
- if (requests.containsKey(new OutstandingRequest(role, hostname))) {
- requested.add(container);
+ if (placedRequests.containsKey(new OutstandingRequest(role, hostname))) {
+ outRequested.add(container);
} else {
- unrequested.add(container);
+ outUnrequested.add(container);
}
}
}
@@ -161,9 +195,9 @@ public class OutstandingRequestTracker {
* @return possibly empty list of hostnames
*/
public synchronized List<NodeInstance> cancelOutstandingRequests(int role) {
- List<NodeInstance> hosts = new ArrayList<NodeInstance>();
+ List<NodeInstance> hosts = new ArrayList<>();
Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
- requests.entrySet().iterator();
+ placedRequests.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<OutstandingRequest, OutstandingRequest> next =
iterator.next();
@@ -176,8 +210,13 @@ public class OutstandingRequestTracker {
}
return hosts;
}
-
+
+ /**
+ * Get a list of outstanding requests. The list is cloned, but the contents
+ * are shared
+ * @return a list of the current outstanding requests
+ */
public synchronized List<OutstandingRequest> listOutstandingRequests() {
- return new ArrayList<OutstandingRequest>(requests.values());
+ return new ArrayList<>(placedRequests.values());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b8750e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 5509cac..eef2b8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -513,9 +513,9 @@ public class RoleHistory {
* Request an instance on a given node.
* An outstanding request is created & tracked, with the
* relevant node entry for that role updated.
- *
+ *<p>
* The role status entries will also be tracked
- *
+ * <p>
* Returns the request that is now being tracked.
* If the node instance is not null, it's details about the role is incremented
*
[14/20] incubator-slider git commit: SLIDER-799 test of escalation
through mock
Posted by st...@apache.org.
SLIDER-799 test of escalation through mock
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/92e7af6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/92e7af6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/92e7af6c
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 92e7af6c6c9b874e2e93118d9315ae266c56fb26
Parents: 9e67161
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 14:00:33 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 14:00:33 2015 +0000
----------------------------------------------------------------------
.../operations/CancelSingleRequest.java | 4 ++
.../slider/server/appmaster/state/AppState.java | 2 +-
.../appmaster/state/OutstandingRequest.java | 16 +++---
.../server/appmaster/state/RoleHistory.java | 4 +-
.../server/appmaster/state/RoleStatus.java | 8 +++
...tRoleHistoryOutstandingRequestTracker.groovy | 55 +++++++++++++++++++-
.../TestRoleHistoryRequestTracking.groovy | 28 +++++-----
.../appmaster/model/mock/MockFactory.groovy | 15 ++++--
8 files changed, 102 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
index 3bb3b09..08eb5bc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -36,6 +36,10 @@ public class CancelSingleRequest extends AbstractRMOperation {
handler.cancelSingleRequest(request);
}
+ public AMRMClient.ContainerRequest getRequest() {
+ return request;
+ }
+
@Override
public String toString() {
return "cancel single request for container at " + request.getPriority().toString();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index a887107..b7ca526 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1203,7 +1203,7 @@ public class AppState {
* @param capability a resource to set up
* @return the request for a new container
*/
- private AMRMClient.ContainerRequest buildContainerResourceAndRequest(
+ public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
RoleStatus role,
Resource capability) {
buildResourceRequirements(role, capability);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 7ad3fbb..799d08e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -140,7 +140,7 @@ public final class OutstandingRequest {
boolean relaxLocality;
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
- boolean usePlacementHistory = role.isStrictPlacement();
+ boolean usePlacementHistory = role.isStrictOrAntiAffinePlacement();
if (!usePlacementHistory) {
// If strict placement does not mandate using placement then check
// that the recent failures on this node is not higher than threshold
@@ -157,7 +157,7 @@ public final class OutstandingRequest {
if (node != null && usePlacementHistory) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = !role.isStrictPlacement();
+ relaxLocality = !role.isStrictOrAntiAffinePlacement();
// tell the node it is in play
node.getOrCreate(roleId);
log.info("Submitting request for container on {}", hosts[0]);
@@ -265,14 +265,16 @@ public final class OutstandingRequest {
result = 31 * result + (hostname != null ? hostname.hashCode() : 0);
return result;
}
-
+
@Override
public String toString() {
- final StringBuilder sb =
- new StringBuilder("OutstandingRequest{");
+ final StringBuilder sb = new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
- sb.append(", node='").append(node).append('\'');
- sb.append(", requestedTime=").append(requestedTimeMillis);
+ sb.append(", node=").append(node);
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
+ sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
+ sb.append(", escalated=").append(escalated);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 7ac5c52..6aefc47 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -835,11 +835,11 @@ public class RoleHistory {
}
/**
- * Get a snapshot of the outstanding request list
+ * Get a snapshot of the outstanding placed request list
* @return a list of the requests outstanding at the time of requesting
*/
@VisibleForTesting
- public synchronized List<OutstandingRequest> getOutstandingRequestList() {
+ public synchronized List<OutstandingRequest> listOutstandingPlacedRequests() {
return outstandingRequests.listOutstandingRequests();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 7ab7dbe..4ee8fad 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -99,6 +99,14 @@ public final class RoleStatus implements Cloneable {
return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT);
}
+ public boolean isAntiAffinePlacement() {
+ return 0 != (getPlacementPolicy() & PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+ }
+
+ public boolean isStrictOrAntiAffinePlacement() {
+ return isStrictPlacement() || isAntiAffinePlacement();
+ }
+
public synchronized int getDesired() {
return desired;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 0143bce..8c79bbf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -18,7 +18,14 @@
package org.apache.slider.server.appmaster.model.history
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockResource
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -70,8 +77,52 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert tracker.resetOutstandingRequests(1).size() == 1
}
+/*
+ @Override
+ AggregateConf buildInstanceDefinition() {
+ def aggregateConf = super.buildInstanceDefinition()
+ def component0 = aggregateConf.resourceOperations.getMandatoryComponent(ROLE0)
+ component0.set(ResourceKeys.COMPONENT_PLACEMENT_POLICY, PlacementPolicy.STRICT)
+ return aggregateConf
+ }
+*/
+
@Test
- public void testEscalation() throws Throwable {
- OutstandingRequest r1 = tracker.newRequest(host1, 0)
+ public void testEscalationOfStrictPlacement() throws Throwable {
+ final def roleStatus = role1Status
+
+
+ ProviderRole role = roleStatus.providerRole
+ assert role.placementPolicy == PlacementPolicy.STRICT;
+ Resource resource = new MockResource()
+
+ appState.buildResourceRequirements(roleStatus, resource)
+
+ // first requst
+ OutstandingRequest r1 = tracker.newRequest(host1, roleStatus.key)
+ final def initialRequest = r1.buildContainerRequest(resource, roleStatus, 0, null)
+ assert r1.issuedRequest != null;
+ assert r1.located
+ assert !r1.escalated
+
+ assert !initialRequest.relaxLocality
+ assert tracker.listOutstandingRequests().size() == 1
+
+ // simulate a few minutes; escalation MUST now be triggered
+ List<AbstractRMOperation> escalations = tracker.escalateOutstandingRequests(180 * 1000)
+
+ assert r1.escalated
+ // two entries
+ assert escalations.size() == 2;
+ final def e1 = escalations[0]
+ assert e1 instanceof CancelSingleRequest
+ CancelSingleRequest cancel = (CancelSingleRequest) e1
+ assert initialRequest == cancel.request
+ final def e2 = escalations[1]
+ assert e2 instanceof ContainerRequestOperation;
+ final def escRequest = (ContainerRequestOperation) e2
+ def req2 = escRequest.request
+ assert req2.relaxLocality
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index d87222d..e593eab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -122,7 +122,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
@@ -130,28 +130,28 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 1
String hostname = requests[0].hostname
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 2, 2)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
@@ -160,22 +160,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
assert !roleHistory.onContainerAllocated(container, 3, 3)
// yet the list is now empty
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -189,17 +189,17 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- assert roleHistory.outstandingRequestList.size() == 2
+ assert roleHistory.listOutstandingPlacedRequests.size() == 2
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 3, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 3)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 311c049..37f4021 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -28,6 +28,7 @@ import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
/**
@@ -42,9 +43,15 @@ class MockFactory implements MockRoles {
0)
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
+ 1,
+ PlacementPolicy.STRICT,
+ 3,
1)
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
+ 2,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 4,
2)
int appIdCount;
int attemptIdCount;
@@ -169,12 +176,12 @@ class MockFactory implements MockRoles {
instance.setResources(newConfTree(r1, r2, r3))
return instance
}
-
-
-
+
def roleMap(int count) {
return [
- (ResourceKeys.COMPONENT_INSTANCES):count.toString()
+ (ResourceKeys.COMPONENT_INSTANCES):count.toString(),
+ (ResourceKeys.COMPONENT_PLACEMENT_POLICY):"${PlacementPolicy.STRICT}".toString()
+
]
}
[02/20] incubator-slider git commit: SLIDER-799 javadoc cleanup
Posted by st...@apache.org.
SLIDER-799 javadoc cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8c8682f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8c8682f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8c8682f5
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 8c8682f5162f17ec47081086465aac6cfcbf9388
Parents: 55815c2
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 3 15:25:52 2015 -0800
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 3 15:26:18 2015 -0800
----------------------------------------------------------------------
.../slider/server/appmaster/state/OutstandingRequest.java | 7 +++----
.../slider/server/appmaster/state/ProviderAppState.java | 4 ++++
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8c8682f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 6acac89..2bc211a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -27,12 +27,11 @@ import org.slf4j.LoggerFactory;
/**
* Tracks an outstanding request. This is used to correlate an allocation response
- * with the
- * node and role used in the request.
- *
+ * with the node and role used in the request.
+ * <p>
* The node identifier may be null -which indicates that a request was made without
* a specific target node
- *
+ * <p>
* Equality and the hash code are based <i>only</i> on the role and hostname,
* which are fixed in the constructor. This means that a simple
* instance constructed with (role, hostname) can be used to look up
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8c8682f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index d237f42..632551e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -39,6 +39,10 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+/**
+ * Implementation of {@link StateAccessForProviders}, which means
+ * state access for providers, web UI and IPC/REST views.
+ */
public class ProviderAppState implements StateAccessForProviders {
[06/20] incubator-slider git commit: Merge branch 'develop' into
feature/SLIDER-799-AM-managed-relax
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-799-AM-managed-relax
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a0d76b74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a0d76b74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a0d76b74
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: a0d76b74836df671e1456ddc92c9cb7a423c7cf1
Parents: 0dd979c 38fed99
Author: Steve Loughran <st...@apache.org>
Authored: Mon Mar 16 12:01:12 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Mar 16 12:01:12 2015 +0000
----------------------------------------------------------------------
app-packages/accumulo/accumulo-slider | 53 -
app-packages/accumulo/accumulo-slider.py | 378 -----
app-packages/accumulo/appConfig-default.json | 1 -
.../accumulo/appConfig-secured-default.json | 1 -
.../accumulo/appConfig-ssl-default.json | 99 ++
.../accumulo/clientInstallConfig-default.json | 6 +
.../accumulo/configuration/accumulo-env.xml | 2 +-
app-packages/accumulo/configuration/client.xml | 5 -
app-packages/accumulo/metainfo.xml | 2 +-
.../accumulo/package/files/accumulo-slider | 59 +
.../accumulo/package/files/accumulo-slider.py | 182 +++
.../accumulo/package/scripts/accumulo_client.py | 81 +-
.../package/scripts/accumulo_configuration.py | 73 +-
.../accumulo/package/scripts/client_params.py | 48 +
app-packages/accumulo/package/scripts/params.py | 21 +-
.../package/templates/accumulo-env.sh.j2 | 42 -
.../package/templates/accumulo-slider-env.sh.j2 | 17 +
.../hadoop-metrics2-accumulo.properties.j2 | 2 +-
app-packages/accumulo/pom.xml | 14 +-
app-packages/accumulo/src/assembly/accumulo.xml | 15 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 6 +-
.../accumulo/AccumuloMonitorSSLIT.groovy | 8 +
.../funtest/accumulo/AccumuloReadWriteIT.groovy | 3 +-
.../accumulo/AccumuloReadWriteSSLIT.groovy | 27 +-
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 86 +-
.../funtest/accumulo/AccumuloScriptIT.groovy | 96 +-
.../funtest/accumulo/AccumuloScriptSSLIT.groovy | 147 ++
.../funtest/accumulo/AccumuloSliderShell.groovy | 7 -
.../slider/funtest/accumulo/CertUtil.java | 275 ----
.../resources/clientInstallConfig-ssl-test.json | 7 +
.../src/test/resources/test_shell_cmd_file | 20 +
app-packages/hbase-win/README.txt | 10 +-
app-packages/hbase/README.md | 11 +-
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 2 +-
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 2 +-
pom.xml | 2 +-
slider-assembly/src/main/scripts/slider | 4 +-
slider-core/pom.xml | 2 +
.../slider/api/SliderClusterProtocol.java | 6 +-
.../org/apache/slider/api/proto/Messages.java | 1503 +++++++++++++++++-
.../slider/api/proto/RestTypeMarshalling.java | 24 +
.../slider/api/proto/SliderClusterAPI.java | 81 +-
.../org/apache/slider/client/SliderClient.java | 120 +-
.../apache/slider/client/SliderClientAPI.java | 2 +-
.../client/ipc/SliderClusterOperations.java | 21 +-
.../slider/common/params/ActionClientArgs.java | 33 +
.../apache/slider/common/params/Arguments.java | 6 +
.../slider/common/params/SliderActions.java | 4 +-
.../apache/slider/common/tools/PortScanner.java | 2 +-
.../providers/AbstractClientProvider.java | 8 +-
.../providers/agent/AgentClientProvider.java | 25 +-
.../providers/agent/AgentProviderService.java | 6 +-
.../server/appmaster/SliderAppMaster.java | 6 +-
.../management/MetricsBindingService.java | 18 +-
.../rpc/SliderClusterProtocolPBImpl.java | 12 +
.../rpc/SliderClusterProtocolProxy.java | 10 +
.../server/appmaster/rpc/SliderIPCService.java | 39 +-
.../appmaster/rpc/SliderRPCSecurityInfo.java | 4 +-
.../services/security/CertificateManager.java | 28 +-
.../src/main/proto/SliderClusterMessages.proto | 11 +
.../src/main/proto/SliderClusterProtocol.proto | 5 +-
.../agent/TestAgentClientProvider2.java | 34 +-
.../AMClientCertStoreRetrievalIT.groovy | 246 +++
63 files changed, 3018 insertions(+), 1052 deletions(-)
----------------------------------------------------------------------
[15/20] incubator-slider git commit: SLIDER-799 code & Test to ensure
that there's no race condition during the escalation scan + execute cycle
with entries being created & issued
Posted by st...@apache.org.
SLIDER-799 code & Test to ensure that there's no race condition during the escalation scan + execute cycle with entries being created & issued
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd2f724
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd2f724
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd2f724
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 3dd2f7244026cedf62b651ee8f97f5d0dd885859
Parents: 92e7af6
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 14:34:18 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 14:34:18 2015 +0000
----------------------------------------------------------------------
.../appmaster/state/OutstandingRequest.java | 14 +++--
.../state/OutstandingRequestTracker.java | 18 ++++---
...tRoleHistoryOutstandingRequestTracker.groovy | 56 +++++++++++++-------
3 files changed, 53 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd2f724/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 799d08e..4fd2933 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -26,8 +26,6 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
/**
@@ -134,7 +132,7 @@ public final class OutstandingRequest {
* @param labelExpression label to satisfy
* @return the request to raise
*/
- public AMRMClient.ContainerRequest buildContainerRequest(
+ public synchronized AMRMClient.ContainerRequest buildContainerRequest(
Resource resource, RoleStatus role, long time, String labelExpression) {
String[] hosts;
boolean relaxLocality;
@@ -187,9 +185,9 @@ public final class OutstandingRequest {
* as the original one, and the same host, but: relaxed placement, and a changed priority
* so as to place it into the relaxed list.
*/
- public AMRMClient.ContainerRequest escalate() {
+ public synchronized AMRMClient.ContainerRequest escalate() {
+ Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued "+ this);
escalated = true;
- Preconditions.checkNotNull(issuedRequest, "issued request");
Priority pri = ContainerPriority.createPriority(roleId, true);
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
@@ -224,8 +222,8 @@ public final class OutstandingRequest {
* @param time time to check against
* @return true if escalation should begin
*/
- public boolean shouldEscalate(long time) {
- return !escalated && escalationTimeoutMillis < time;
+ public synchronized boolean shouldEscalate(long time) {
+ return issuedRequest != null && !escalated && escalationTimeoutMillis < time;
}
/**
@@ -267,7 +265,7 @@ public final class OutstandingRequest {
}
@Override
- public String toString() {
+ public synchronized String toString() {
final StringBuilder sb = new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
sb.append(", node=").append(node);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd2f724/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 93db3fa..48f6e57 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -245,14 +245,18 @@ public class OutstandingRequestTracker {
List<AbstractRMOperation> operations = new ArrayList<>();
for (OutstandingRequest outstandingRequest : placedRequests.values()) {
- if (outstandingRequest.shouldEscalate(now)) {
+ synchronized (outstandingRequest) {
+ // sync escalation check with operation so that nothing can happen to state
+ // of the request during the escalation
+ if (outstandingRequest.shouldEscalate(now)) {
- // time to escalate
- CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.issuedRequest);
- operations.add(cancel);
- AMRMClient.ContainerRequest escalated =
- outstandingRequest.escalate();
- operations.add(new ContainerRequestOperation(escalated));
+ // time to escalate
+ CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.issuedRequest);
+ operations.add(cancel);
+ AMRMClient.ContainerRequest escalated =
+ outstandingRequest.escalate();
+ operations.add(new ContainerRequestOperation(escalated));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd2f724/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 8c79bbf..2fe6763 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -29,6 +29,7 @@ import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@@ -88,30 +89,32 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
*/
@Test
- public void testEscalationOfStrictPlacement() throws Throwable {
- final def roleStatus = role1Status
-
-
- ProviderRole role = roleStatus.providerRole
- assert role.placementPolicy == PlacementPolicy.STRICT;
- Resource resource = new MockResource()
-
- appState.buildResourceRequirements(roleStatus, resource)
-
- // first requst
- OutstandingRequest r1 = tracker.newRequest(host1, roleStatus.key)
- final def initialRequest = r1.buildContainerRequest(resource, roleStatus, 0, null)
- assert r1.issuedRequest != null;
- assert r1.located
- assert !r1.escalated
-
+ public void testEscalation() throws Throwable {
+ ProviderRole providerRole1 = role1Status.providerRole
+ assert providerRole1.placementPolicy == PlacementPolicy.STRICT;
+ // first request
+ final def (res1, outstanding1) = newRequest(role1Status)
+ final def initialRequest = outstanding1.buildContainerRequest(res1, role1Status, 0, null)
+ assert outstanding1.issuedRequest != null;
+ assert outstanding1.located
+ assert !outstanding1.escalated
assert !initialRequest.relaxLocality
assert tracker.listOutstandingRequests().size() == 1
- // simulate a few minutes; escalation MUST now be triggered
- List<AbstractRMOperation> escalations = tracker.escalateOutstandingRequests(180 * 1000)
+ // second. This one doesn't get launched. This is to verify that the escalation
+ // process skips entries which are in the list but have not been issued.
+ // ...which can be a race condition between request issuance & escalation.
+ // (not one observed outside test authoring, but retained for completeness)
+ assert role2Status.placementPolicy == PlacementPolicy.ANTI_AFFINITY_REQUIRED
+ def (res2, outstanding2) = newRequest(role2Status)
+
+ // simulate some time escalation of role 1 MUST now be triggered
+ List<AbstractRMOperation> escalations =
+ tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000 + 500 )
+
+ assert outstanding1.escalated
+ assert !outstanding2.escalated
- assert r1.escalated
// two entries
assert escalations.size() == 2;
final def e1 = escalations[0]
@@ -124,5 +127,18 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
def req2 = escRequest.request
assert req2.relaxLocality
+ def (res3, outstanding3) = newRequest(role2Status)
+ outstanding3.buildContainerRequest(res3, role2Status, 0, null)
+
+ List<AbstractRMOperation> escalations2 =
+ tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000 + 500)
+ assert escalations2.size() == 0
+ }
+
+ public def newRequest(RoleStatus r) {
+ final Resource res2 = new MockResource()
+ appState.buildResourceRequirements(r, res2)
+ final OutstandingRequest outstanding2 = tracker.newRequest(host1, r.key)
+ return [res2, outstanding2]
}
}
[11/20] incubator-slider git commit: SLIDER-799 rename
cancelOutstandingRequests to resetOutstandingRequests -as all it does is
reset the data structures
Posted by st...@apache.org.
SLIDER-799 rename cancelOutstandingRequests to resetOutstandingRequests
-as all it does is reset the data structures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1624bdfb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1624bdfb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1624bdfb
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 1624bdfbe75194b78ef58cc6f68cb67d0a447e3c
Parents: 06e8db3
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 10:30:19 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 10:30:19 2015 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/SliderAppMaster.java | 2 +-
.../appmaster/state/OutstandingRequestTracker.java | 4 ++--
.../slider/server/appmaster/state/RoleHistory.java | 4 ++--
.../TestRoleHistoryOutstandingRequestTracker.groovy | 14 +++++++++-----
4 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1624bdfb/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a93c60b..fd420fd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -863,7 +863,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// build up environment variables that the AM wants set in every container
// irrespective of provider and role.
- envVars = new HashMap<String, String>();
+ envVars = new HashMap<>();
if (hadoop_user_name != null) {
envVars.put(HADOOP_USER_NAME, hadoop_user_name);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1624bdfb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index eb58e74..93db3fa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -202,13 +202,13 @@ public class OutstandingRequestTracker {
/**
- * Cancel all outstanding requests for a role: return the hostnames
+ * Reset list all outstanding requests for a role: return the hostnames
* of any canceled requests
*
* @param role role to cancel
* @return possibly empty list of hostnames
*/
- public synchronized List<NodeInstance> cancelOutstandingRequests(int role) {
+ public synchronized List<NodeInstance> resetOutstandingRequests(int role) {
List<NodeInstance> hosts = new ArrayList<>();
Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
placedRequests.entrySet().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1624bdfb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0d4de2d..7ac5c52 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -645,10 +645,10 @@ public class RoleHistory {
boolean requestFound =
outstandingRequests.onContainerAllocated(role, hostname);
if (desiredCount <= actualCount) {
- // all oustanding requests have been satisfied
+ // all outstanding requests have been satisfied
// tag nodes as available
List<NodeInstance>
- hosts = outstandingRequests.cancelOutstandingRequests(role);
+ hosts = outstandingRequests.resetOutstandingRequests(role);
if (!hosts.isEmpty()) {
//add the list
log.info("Adding {} hosts for role {}", hosts.size(), role);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1624bdfb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 79a9bd6..0143bce 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -55,19 +55,23 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
}
@Test
- public void testCancelEntries() throws Throwable {
+ public void testResetEntries() throws Throwable {
OutstandingRequest r1 = tracker.newRequest(host1, 0)
OutstandingRequest r2 = tracker.newRequest(host2, 0)
OutstandingRequest r3 = tracker.newRequest(host1, 1)
- List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
+ List<NodeInstance> canceled = tracker.resetOutstandingRequests(0)
assert canceled.size() == 2
assert canceled.contains(host1)
assert canceled.contains(host2)
assert tracker.lookup(1, "host1")
assert !tracker.lookup(0, "host1")
- canceled = tracker.cancelOutstandingRequests(0)
+ canceled = tracker.resetOutstandingRequests(0)
assert canceled.size() == 0
- assert tracker.cancelOutstandingRequests(1).size() == 1
+ assert tracker.resetOutstandingRequests(1).size() == 1
+ }
+
+ @Test
+ public void testEscalation() throws Throwable {
+ OutstandingRequest r1 = tracker.newRequest(host1, 0)
}
-
}
[05/20] incubator-slider git commit: SLIDER-799: we'll need a feed on
escalation
Posted by st...@apache.org.
SLIDER-799: we'll need a feed on escalation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0dd979cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0dd979cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0dd979cc
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 0dd979cc9f13e1e539fafe06cc695472a94b758f
Parents: 7141d93
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 5 17:35:36 2015 -0800
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 5 17:35:36 2015 -0800
----------------------------------------------------------------------
.../slider/server/appmaster/state/OutstandingRequest.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0dd979cc/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 3563a29..c6a9cff 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -56,11 +56,16 @@ public final class OutstandingRequest {
public final String hostname;
/**
- * requested time -only valid after buildContainerRequest(Resource, long)}
+ * requested time -only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
*/
public long requestedTime;
/**
+ * Has the placement request been escalated by cancel and re-request
+ */
+ public boolean escalated;
+
+ /**
* Create a request
* @param roleId role
* @param node node -can be null
[17/20] incubator-slider git commit: SLIDER-799 track outcome of
allocation: whether an assignment was "open", "placed", or "escalated";
this info is included in serialized/JSON views of container state so can be
retrieved by client APIs
Posted by st...@apache.org.
SLIDER-799 track outcome of allocation: whether an assignment was "open", "placed", or "escalated"; this info is included in serialized/JSON views of container state so can be retrieved by client APIs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7d9a9e94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7d9a9e94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7d9a9e94
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 7d9a9e942ead8343bc4e2c52419c1b258292ca15
Parents: ad41b24
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 24 11:05:39 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 24 11:05:39 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 8 +-
.../org/apache/slider/api/proto/Messages.java | 223 ++++++++++++++++---
.../slider/api/proto/RestTypeMarshalling.java | 8 +-
.../slider/api/types/ContainerInformation.java | 4 +-
.../apache/slider/providers/ProviderRole.java | 2 +-
.../server/appmaster/RoleLaunchService.java | 38 ++--
.../server/appmaster/SliderAppMaster.java | 8 +-
.../slider/server/appmaster/state/AppState.java | 14 +-
.../state/ContainerAllocationOutcome.java | 29 +++
.../appmaster/state/ContainerAssignment.java | 31 ++-
.../server/appmaster/state/NodeEntry.java | 7 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../state/OutstandingRequestTracker.java | 12 +-
.../server/appmaster/state/RoleHistory.java | 13 +-
.../server/appmaster/state/RoleInstance.java | 21 +-
.../resources/LiveContainersRefresher.java | 2 +-
.../src/main/proto/SliderClusterMessages.proto | 1 +
...tRoleHistoryOutstandingRequestTracker.groovy | 4 +-
.../TestRoleHistoryRequestTracking.groovy | 79 ++++---
19 files changed, 385 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 94ce681..9066a52 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -142,10 +142,10 @@ public interface ResourceKeys {
/**
- * Time in seconds to relax placement delay
+ * Time in seconds to escalate placement delay
*/
- String PLACEMENT_RELAX_DELAY =
- "yarn.placement.relax.seconds";
+ String PLACEMENT_ESCALATE_DELAY =
+ "yarn.placement.escalate.seconds";
/**
* Time to have a strict placement policy outstanding before
@@ -156,7 +156,7 @@ public interface ResourceKeys {
* </ol>
*
*/
- int DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS = 30;
+ int DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS = 30;
/**
* Log aggregation include, exclude patterns
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
index 53c7d45..44771e5 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -15438,6 +15438,21 @@ public final class Messages {
*/
com.google.protobuf.ByteString
getHostURLBytes();
+
+ // optional string placement = 12;
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ boolean hasPlacement();
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ java.lang.String getPlacement();
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ com.google.protobuf.ByteString
+ getPlacementBytes();
}
/**
* Protobuf type {@code org.apache.slider.api.ContainerInformationProto}
@@ -15553,6 +15568,11 @@ public final class Messages {
hostURL_ = input.readBytes();
break;
}
+ case 98: {
+ bitField0_ |= 0x00000400;
+ placement_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -15921,6 +15941,49 @@ public final class Messages {
}
}
+ // optional string placement = 12;
+ public static final int PLACEMENT_FIELD_NUMBER = 12;
+ private java.lang.Object placement_;
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public boolean hasPlacement() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public java.lang.String getPlacement() {
+ java.lang.Object ref = placement_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ placement_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public com.google.protobuf.ByteString
+ getPlacementBytes() {
+ java.lang.Object ref = placement_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ placement_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
private void initFields() {
containerId_ = "";
component_ = "";
@@ -15933,6 +15996,7 @@ public final class Messages {
output_ = com.google.protobuf.LazyStringArrayList.EMPTY;
host_ = "";
hostURL_ = "";
+ placement_ = "";
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -15979,6 +16043,9 @@ public final class Messages {
if (((bitField0_ & 0x00000200) == 0x00000200)) {
output.writeBytes(11, getHostURLBytes());
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBytes(12, getPlacementBytes());
+ }
getUnknownFields().writeTo(output);
}
@@ -16037,6 +16104,10 @@ public final class Messages {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(11, getHostURLBytes());
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(12, getPlacementBytes());
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -16112,6 +16183,11 @@ public final class Messages {
result = result && getHostURL()
.equals(other.getHostURL());
}
+ result = result && (hasPlacement() == other.hasPlacement());
+ if (hasPlacement()) {
+ result = result && getPlacement()
+ .equals(other.getPlacement());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -16169,6 +16245,10 @@ public final class Messages {
hash = (37 * hash) + HOSTURL_FIELD_NUMBER;
hash = (53 * hash) + getHostURL().hashCode();
}
+ if (hasPlacement()) {
+ hash = (37 * hash) + PLACEMENT_FIELD_NUMBER;
+ hash = (53 * hash) + getPlacement().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -16305,6 +16385,8 @@ public final class Messages {
bitField0_ = (bitField0_ & ~0x00000200);
hostURL_ = "";
bitField0_ = (bitField0_ & ~0x00000400);
+ placement_ = "";
+ bitField0_ = (bitField0_ & ~0x00000800);
return this;
}
@@ -16379,6 +16461,10 @@ public final class Messages {
to_bitField0_ |= 0x00000200;
}
result.hostURL_ = hostURL_;
+ if (((from_bitField0_ & 0x00000800) == 0x00000800)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.placement_ = placement_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -16445,6 +16531,11 @@ public final class Messages {
hostURL_ = other.hostURL_;
onChanged();
}
+ if (other.hasPlacement()) {
+ bitField0_ |= 0x00000800;
+ placement_ = other.placement_;
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -17100,6 +17191,80 @@ public final class Messages {
return this;
}
+ // optional string placement = 12;
+ private java.lang.Object placement_ = "";
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public boolean hasPlacement() {
+ return ((bitField0_ & 0x00000800) == 0x00000800);
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public java.lang.String getPlacement() {
+ java.lang.Object ref = placement_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ placement_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public com.google.protobuf.ByteString
+ getPlacementBytes() {
+ java.lang.Object ref = placement_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ placement_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public Builder setPlacement(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000800;
+ placement_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public Builder clearPlacement() {
+ bitField0_ = (bitField0_ & ~0x00000800);
+ placement_ = getDefaultInstance().getPlacement();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string placement = 12;</code>
+ */
+ public Builder setPlacementBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000800;
+ placement_ = value;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:org.apache.slider.api.ContainerInformationProto)
}
@@ -26869,38 +27034,38 @@ public final class Messages {
"ed\030\t \001(\005\022\021\n\tcompleted\030\n \001(\005\022\026\n\016totalRequ" +
"ested\030\013 \001(\005\022\026\n\016failureMessage\030\014 \001(\t\022\027\n\017p" +
"lacementPolicy\030\r \001(\005\022\022\n\ncontainers\030\016 \003(\t" +
- "\"\341\001\n\031ContainerInformationProto\022\023\n\013contai" +
+ "\"\364\001\n\031ContainerInformationProto\022\023\n\013contai" +
"nerId\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010releas" +
"ed\030\003 \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030\005 \001(" +
"\005\022\023\n\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime\030\007 \001" +
"(\003\022\021\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(\t\022\014\n" +
- "\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\"N\n\024PingInfo" +
- "rmationProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 \001(\t",
- "\022\014\n\004body\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\026\n\024GetModel" +
- "RequestProto\"\035\n\033GetModelDesiredRequestPr" +
- "oto\"$\n\"GetModelDesiredAppconfRequestProt" +
- "o\"&\n$GetModelDesiredResourcesRequestProt" +
- "o\"%\n#GetModelResolvedAppconfRequestProto" +
- "\"\'\n%GetModelResolvedResourcesRequestProt" +
- "o\"#\n!GetModelLiveResourcesRequestProto\"\037" +
- "\n\035GetLiveContainersRequestProto\"u\n\036GetLi" +
- "veContainersResponseProto\022\r\n\005names\030\001 \003(\t" +
- "\022D\n\ncontainers\030\002 \003(\01320.org.apache.slider",
- ".api.ContainerInformationProto\"3\n\034GetLiv" +
- "eContainerRequestProto\022\023\n\013containerId\030\001 " +
- "\002(\t\"\037\n\035GetLiveComponentsRequestProto\"u\n\036" +
- "GetLiveComponentsResponseProto\022\r\n\005names\030" +
- "\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org.apache.s" +
- "lider.api.ComponentInformationProto\",\n\034G" +
- "etLiveComponentRequestProto\022\014\n\004name\030\001 \002(" +
- "\t\"$\n\"GetApplicationLivenessRequestProto\"" +
- "\023\n\021EmptyPayloadProto\" \n\020WrappedJsonProto" +
- "\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificateStoreReq",
- "uestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013requester" +
- "Id\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004 \002(\t" +
- "\"1\n GetCertificateStoreResponseProto\022\r\n\005" +
- "store\030\001 \002(\014B-\n\033org.apache.slider.api.pro" +
- "toB\010Messages\210\001\001\240\001\001"
+ "\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplacemen" +
+ "t\030\014 \001(\t\"N\n\024PingInformationProto\022\014\n\004text\030",
+ "\001 \001(\t\022\014\n\004verb\030\002 \001(\t\022\014\n\004body\030\003 \001(\t\022\014\n\004tim" +
+ "e\030\004 \001(\003\"\026\n\024GetModelRequestProto\"\035\n\033GetMo" +
+ "delDesiredRequestProto\"$\n\"GetModelDesire" +
+ "dAppconfRequestProto\"&\n$GetModelDesiredR" +
+ "esourcesRequestProto\"%\n#GetModelResolved" +
+ "AppconfRequestProto\"\'\n%GetModelResolvedR" +
+ "esourcesRequestProto\"#\n!GetModelLiveReso" +
+ "urcesRequestProto\"\037\n\035GetLiveContainersRe" +
+ "questProto\"u\n\036GetLiveContainersResponseP" +
+ "roto\022\r\n\005names\030\001 \003(\t\022D\n\ncontainers\030\002 \003(\0132",
+ "0.org.apache.slider.api.ContainerInforma" +
+ "tionProto\"3\n\034GetLiveContainerRequestProt" +
+ "o\022\023\n\013containerId\030\001 \002(\t\"\037\n\035GetLiveCompone" +
+ "ntsRequestProto\"u\n\036GetLiveComponentsResp" +
+ "onseProto\022\r\n\005names\030\001 \003(\t\022D\n\ncomponents\030\002" +
+ " \003(\01320.org.apache.slider.api.ComponentIn" +
+ "formationProto\",\n\034GetLiveComponentReques" +
+ "tProto\022\014\n\004name\030\001 \002(\t\"$\n\"GetApplicationLi" +
+ "venessRequestProto\"\023\n\021EmptyPayloadProto\"" +
+ " \n\020WrappedJsonProto\022\014\n\004json\030\001 \002(\t\"h\n\037Get",
+ "CertificateStoreRequestProto\022\020\n\010hostname" +
+ "\030\001 \001(\t\022\023\n\013requesterId\030\002 \002(\t\022\020\n\010password\030" +
+ "\003 \002(\t\022\014\n\004type\030\004 \002(\t\"1\n GetCertificateSto" +
+ "reResponseProto\022\r\n\005store\030\001 \002(\014B-\n\033org.ap" +
+ "ache.slider.api.protoB\010Messages\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -27050,7 +27215,7 @@ public final class Messages {
internal_static_org_apache_slider_api_ContainerInformationProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_ContainerInformationProto_descriptor,
- new java.lang.String[] { "ContainerId", "Component", "Released", "State", "ExitCode", "Diagnostics", "CreateTime", "StartTime", "Output", "Host", "HostURL", });
+ new java.lang.String[] { "ContainerId", "Component", "Released", "State", "ExitCode", "Diagnostics", "CreateTime", "StartTime", "Output", "Host", "HostURL", "Placement", });
internal_static_org_apache_slider_api_PingInformationProto_descriptor =
getDescriptor().getMessageTypes().get(24);
internal_static_org_apache_slider_api_PingInformationProto_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 84a950d..4264582 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -157,7 +157,10 @@ public class RestTypeMarshalling {
info.startTime = wire.getStartTime();
info.output = wire.getOutputList().toArray(
new String[wire.getOutputCount()]
- );
+ );
+ if (wire.hasPlacement()) {
+ info.placement = wire.getPlacement();
+ }
return info;
}
@@ -199,6 +202,9 @@ public class RestTypeMarshalling {
if (info.released != null) {
builder.setReleased(info.released);
}
+ if (info.placement != null) {
+ builder.setPlacement(info.placement);
+ }
builder.setStartTime(info.startTime);
builder.setState(info.state);
return builder.build();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
index ccadd9c..8c26bfe 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.registry.client.binding.JsonSerDeser;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
-
/**
* Serializable version of component instance data
*/
@@ -41,6 +40,7 @@ public class ContainerInformation {
public String host;
public String hostURL;
+ public String placement;
/**
* What is the tail output from the executed process (or [] if not started
* or the log cannot be picked up
@@ -50,7 +50,7 @@ public class ContainerInformation {
@Override
public String toString() {
JsonSerDeser<ContainerInformation> serDeser =
- new JsonSerDeser<ContainerInformation>(
+ new JsonSerDeser<>(
ContainerInformation.class);
return serDeser.toString(this);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 7caae48..294fe89 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -38,7 +38,7 @@ public final class ProviderRole {
id,
PlacementPolicy.DEFAULT,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS);
+ ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 4333a09..df65ff4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -32,6 +32,7 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
@@ -121,16 +122,15 @@ public class RoleLaunchService
* @param role role
* @param clusterSpec cluster spec to use for template
*/
- public void launchRole(Container container,
- RoleStatus role,
+ public void launchRole(ContainerAssignment assignment,
AggregateConf clusterSpec) {
+ RoleStatus role = assignment.role;
String roleName = role.getName();
// prelaunch safety check
Preconditions.checkArgument(provider.isSupportedRole(roleName));
RoleLaunchService.RoleLauncher launcher =
- new RoleLaunchService.RoleLauncher(container,
- role.getProviderRole(),
- clusterSpec,
+ new RoleLaunchService.RoleLauncher(assignment,
+ clusterSpec,
clusterSpec.getResourceOperations().getOrAddComponent(roleName),
clusterSpec.getAppConfOperations().getOrAddComponent(roleName));
execute(launcher);
@@ -141,27 +141,30 @@ public class RoleLaunchService
*/
private class RoleLauncher implements Runnable {
+ private final ContainerAssignment assignment;
// Allocated container
public final Container container;
- public final String containerRole;
+ public final String containerRole;
private final MapOperations resourceComponent;
private final MapOperations appComponent;
private final AggregateConf instanceDefinition;
public final ProviderRole role;
private Exception raisedException;
- public RoleLauncher(Container container,
- ProviderRole role,
- AggregateConf instanceDefinition,
- MapOperations resourceComponent,
- MapOperations appComponent) {
- assert container != null;
- assert role != null;
+ public RoleLauncher(ContainerAssignment assignment,
+ AggregateConf instanceDefinition,
+ MapOperations resourceComponent,
+ MapOperations appComponent) {
+ this.assignment = assignment;
+ this.container = assignment.container;
+ RoleStatus roleStatus = assignment.role;
+
assert resourceComponent != null;
assert appComponent != null;
- this.container = container;
- this.containerRole = role.name;
- this.role = role;
+ ProviderRole providerRole = roleStatus.getProviderRole();
+ assert providerRole != null;
+ this.containerRole = providerRole.name;
+ this.role = providerRole;
this.resourceComponent = resourceComponent;
this.appComponent = appComponent;
this.instanceDefinition = instanceDefinition;
@@ -228,8 +231,7 @@ public class RoleLaunchService
}
log.info("Container launch delay for {} set to {} seconds",
role.name, delay);
- actionQueue.schedule(new ActionStartContainer("starting "
- + containerRole,
+ actionQueue.schedule(new ActionStartContainer("starting " + containerRole,
container,
containerLauncher.completeContainerLaunch(),
instance,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index fd420fd..ab6b55c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1567,17 +1567,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override //AMRMClientAsync
public void onContainersAllocated(List<Container> allocatedContainers) {
LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size());
- List<ContainerAssignment> assignments = new ArrayList<ContainerAssignment>();
- List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
+ List<ContainerAssignment> assignments = new ArrayList<>();
+ List<AbstractRMOperation> operations = new ArrayList<>();
//app state makes all the decisions
appState.onContainersAllocated(allocatedContainers, assignments, operations);
//for each assignment: instantiate that role
for (ContainerAssignment assignment : assignments) {
- RoleStatus role = assignment.role;
- Container container = assignment.container;
- launchService.launchRole(container, role, getInstanceDefinition());
+ launchService.launchRole(assignment, getInstanceDefinition());
}
//for all the operations, exec them
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 0f07ee9..20e2fc0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -655,8 +655,8 @@ public class AppState {
ResourceKeys.COMPONENT_PLACEMENT_POLICY,
placementOpt, 0, 0, -1);
int placementTimeout =
- component.getOptionInt(ResourceKeys.PLACEMENT_RELAX_DELAY,
- ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS);
+ component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY,
+ ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
ProviderRole newRole = new ProviderRole(name,
priority,
placement,
@@ -1135,7 +1135,7 @@ public class AppState {
* @return the map of Role name to list of Cluster Nodes
*/
public synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
- Map<String, Map<String, ClusterNode>> map = new HashMap<String, Map<String, ClusterNode>>();
+ Map<String, Map<String, ClusterNode>> map = new HashMap<>();
for (RoleInstance node : getLiveNodes().values()) {
Map<String, ClusterNode> containers = map.get(node.role);
@@ -2090,7 +2090,7 @@ public class AppState {
/**
* Event handler for allocated containers: builds up the lists
* of assignment actions (what to run where), and possibly
- * a list of release operations
+ * a list of operations to perform
* @param allocatedContainers the containers allocated
* @param assignments the assignments of roles to containers
* @param releaseOperations any release operations
@@ -2121,7 +2121,9 @@ public class AppState {
//look for (race condition) where we get more back than we asked
desired = role.getDesired();
- roleHistory.onContainerAllocated( container, desired, allocated );
+ ContainerAllocationOutcome outcome = roleHistory.onContainerAllocated(container,
+ desired,
+ allocated);
if (allocated > desired) {
log.info("Discarding surplus container {} on {}", cid,
@@ -2146,7 +2148,7 @@ public class AppState {
container.getNodeId().getPort()
);
- assignments.add(new ContainerAssignment(container, role));
+ assignments.add(new ContainerAssignment(container, role, outcome));
//add to the history
roleHistory.onContainerAssigned(container);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
new file mode 100644
index 0000000..6639300
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+/**
+ * Outcome of the assignment
+ */
+public enum ContainerAllocationOutcome {
+ Unallocated,
+ Open,
+ Placed,
+ Escalated
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
index 3be3777..3e8a3c3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
@@ -20,14 +20,41 @@ package org.apache.slider.server.appmaster.state;
import org.apache.hadoop.yarn.api.records.Container;
+/**
+ * Static assignment structure
+ */
public class ContainerAssignment {
-
+
+ /**
+ * Container that has been allocated
+ */
public final Container container;
+
+ /**
+ * Role to assign to it
+ */
public final RoleStatus role;
+ /**
+ * Placement outcome: was this from history or not
+ */
+ public final ContainerAllocationOutcome placement;
+
public ContainerAssignment(Container container,
- RoleStatus role) {
+ RoleStatus role,
+ ContainerAllocationOutcome placement) {
this.container = container;
this.role = role;
+ this.placement = placement;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("ContainerAssignment{");
+ sb.append("container=").append(container);
+ sb.append(", role=").append(role);
+ sb.append(", placement=").append(placement);
+ sb.append('}');
+ return sb.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index c2e203a..0aa2d42 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -21,7 +21,7 @@ package org.apache.slider.server.appmaster.state;
/**
* Information about the state of a role on a specific node instance.
* No fields are synchronized; sync on the instance to work with it
- *
+ <p>
The two fields `releasing` and `requested` are used to track the ongoing
state of YARN requests; they do not need to be persisted across stop/start
cycles. They may be relevant across AM restart, but without other data
@@ -29,10 +29,11 @@ package org.apache.slider.server.appmaster.state;
it was restarted. The strategy will be to ignore unexpected allocation
responses (which may come from pre-restart) requests, while treating
unexpected container release responses as failures.
-
+ <p>
The `active` counter is only decremented after a container release response
has been received.
-
+ <p>
+
Accesses are synchronized.
*/
public class NodeEntry {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 68c8a15..ed039f9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -26,7 +26,7 @@ import java.util.ListIterator;
/**
* A node instance -stores information about a node in the cluster.
- *
+ * <p>
* Operations on the array/set of roles are synchronized.
*/
public class NodeInstance {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 959cb1f..e226a22 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -103,18 +103,22 @@ public class OutstandingRequestTracker {
* from the {@link #placedRequests} structure.
* @param role role index
* @param hostname hostname
- * @return true if an entry was found and removed
+ * @return the allocation outcome
*/
- public synchronized boolean onContainerAllocated(int role, String hostname) {
+ public synchronized ContainerAllocationOutcome onContainerAllocated(int role, String hostname) {
OutstandingRequest request =
placedRequests.remove(new OutstandingRequest(role, hostname));
if (request == null) {
- return false;
+ // not in the list; this is an open placement
+ return ContainerAllocationOutcome.Open;
} else {
//satisfied request
request.completed();
+ // derive outcome from status of tracked request
+ return request.isEscalated()
+ ? ContainerAllocationOutcome.Escalated
+ : ContainerAllocationOutcome.Placed;
}
- return true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index a1b54c7..99108fe 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -636,20 +636,21 @@ public class RoleHistory {
* @param container container
* @param desiredCount desired #of instances
* @param actualCount current count of instances
- * @return true if an entry was found and dropped
+ * @return The allocation outcome
*/
- public synchronized boolean onContainerAllocated(Container container, int desiredCount, int actualCount) {
+ public synchronized ContainerAllocationOutcome onContainerAllocated(Container container,
+ int desiredCount,
+ int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
LinkedList<NodeInstance> nodeInstances =
getOrCreateNodesForRoleId(role);
- boolean requestFound =
- outstandingRequests.onContainerAllocated(role, hostname);
+ ContainerAllocationOutcome outcome = outstandingRequests.onContainerAllocated(role, hostname);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
// tag nodes as available
List<NodeInstance>
- hosts = outstandingRequests.resetOutstandingRequests(role);
+ hosts = outstandingRequests.resetOutstandingRequests(role);
if (!hosts.isEmpty()) {
//add the list
log.info("Adding {} hosts for role {}", hosts.size(), role);
@@ -657,7 +658,7 @@ public class RoleHistory {
sortAvailableNodeList(role);
}
}
- return requestFound;
+ return outcome;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 4f9b222..726ee08 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -19,12 +19,12 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.types.ContainerInformation;
@@ -95,14 +95,23 @@ public final class RoleInstance implements Cloneable {
public String host;
public String hostURL;
+ public ContainerAllocationOutcome placement;
/**
* A list of registered endpoints.
*/
private List<Endpoint> endpoints =
- new ArrayList<Endpoint>(2);
+ new ArrayList<>(2);
+ public RoleInstance(ContainerAssignment assignment) {
+ this(assignment.container);
+ placement = assignment.placement;
+ }
+ /**
+ * Create an instance to track an allocated container
+ * @param container a container which must be non null, and have a non-null Id field.
+ */
public RoleInstance(Container container) {
Preconditions.checkNotNull(container, "Null container");
Preconditions.checkState(container.getId() != null,
@@ -140,6 +149,7 @@ public final class RoleInstance implements Cloneable {
sb.append(", host=").append(host);
sb.append(", hostURL=").append(hostURL);
sb.append(", state=").append(state);
+ sb.append(", placement=").append(placement);
sb.append(", exitCode=").append(exitCode);
sb.append(", command='").append(command).append('\'');
sb.append(", diagnostics='").append(diagnostics).append('\'');
@@ -288,6 +298,9 @@ public final class RoleInstance implements Cloneable {
info.host = host;
info.hostURL = hostURL;
info.released = released ? Boolean.TRUE : null;
+ if (placement != null) {
+ info.placement = placement.toString();
+ }
if (output != null) {
info.output = output;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
index 276e8cc..68bd8a2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
@@ -42,7 +42,7 @@ public class LiveContainersRefresher implements ResourceRefresher<Map<String, Co
Exception {
List<RoleInstance> containerList = state.cloneOwnedContainerList();
- Map<String, ContainerInformation> map = new HashMap<String, ContainerInformation>();
+ Map<String, ContainerInformation> map = new HashMap<>();
for (RoleInstance instance : containerList) {
ContainerInformation serialized = instance.serialize();
map.put(serialized.containerId, serialized);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
index 9f8f20f..730a49b 100644
--- a/slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -244,6 +244,7 @@ message ContainerInformationProto {
repeated string output = 9;
optional string host = 10;
optional string hostURL = 11;
+ optional string placement = 12;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 97d970d..c30537a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -27,7 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -59,7 +59,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
tracker.newRequest(host1, 0)
tracker.newRequest(host2, 0)
tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1")
+ assert tracker.onContainerAllocated(1, "host1") == ContainerAllocationOutcome.Placed
assert !tracker.lookup(1, "host1")
assert tracker.lookup(0, "host1")
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index e84dfce..bab6233 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -24,6 +24,7 @@ import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -38,7 +39,7 @@ import org.junit.Test
class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
String roleName = "test"
-
+
NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
@@ -52,12 +53,12 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
ProviderRole provRole = new ProviderRole(roleName, 0)
RoleStatus roleStatus = new RoleStatus(provRole)
-
+
@Override
String getTestName() {
return "TestRoleHistoryAvailableList"
}
-
+
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
@@ -69,16 +70,16 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
assertListEquals([age3Active0, age2Active0], available0)
}
-
+
@Test
public void testRequestedNodeOffList() throws Throwable {
List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
assert age3Active0 == ni
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource,
- "")
+ roleStatus,
+ resource,
+ "")
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
assertListEquals([age2Active0], a2)
}
@@ -118,7 +119,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
assertListEquals([age2Active0], a2)
}
-
+
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
@@ -126,7 +127,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
-
+
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
@@ -136,10 +137,26 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
- assert roleHistory.onContainerAllocated(container , 2, 1)
+ assertOnContainerAllocated(container, 2, 1)
+ assertNoOutstandingRequests()
+ }
+
+ public void assertOnContainerAllocated(MockContainer c1, int p1, int p2) {
+ assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2)
+ }
+
+ public void assertOnContainerAllocationOpen(MockContainer c1, int p1, int p2) {
+ assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2)
+ }
+
+ def assertNoOutstandingRequests() {
assert roleHistory.listOutstandingPlacedRequests().empty
}
-
+
+ public void assertOutstandingPlacedRequests(int i) {
+ assert roleHistory.listOutstandingPlacedRequests().size() == i
+ }
+
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
@@ -147,14 +164,13 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
- assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.listOutstandingPlacedRequests().size() == 1
+ assertOnContainerAllocated(container, 2, 1)
+ assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
- assert roleHistory.onContainerAllocated(container, 2, 2)
- assert roleHistory.listOutstandingPlacedRequests().empty
+ assertOnContainerAllocated(container, 2, 2)
+ assertNoOutstandingRequests()
}
-
@Test
public void testThreeRequestsOneUnsatisified() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
@@ -163,19 +179,19 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
- assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.listOutstandingPlacedRequests().size() == 1
-
+ assertOnContainerAllocated(container, 2, 1)
+ assertOutstandingPlacedRequests(1)
+
container = factory.newContainer(req3, "three")
- assert !roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.listOutstandingPlacedRequests().size() == 1
-
+ assertOnContainerAllocationOpen(container, 3, 2)
+ assertOutstandingPlacedRequests(1)
+
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
- assert !roleHistory.onContainerAllocated(container, 3, 3)
+ assertOnContainerAllocationOpen(container, 3, 3)
// yet the list is now empty
- assert roleHistory.listOutstandingPlacedRequests().empty
+ assertNoOutstandingRequests()
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -183,23 +199,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
}
-
@Test
public void testThreeRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- assert roleHistory.listOutstandingPlacedRequests().size() == 2
+ assertOutstandingPlacedRequests(2)
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
- assert roleHistory.onContainerAllocated(container , 3, 1)
- assert roleHistory.listOutstandingPlacedRequests().size() == 1
+ assertOnContainerAllocated(container, 3, 1)
+ assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
- assert roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.listOutstandingPlacedRequests().empty
+ assertOnContainerAllocated(container, 3, 2)
+ assertNoOutstandingRequests()
container = factory.newContainer(req3, "three")
- assert !roleHistory.onContainerAllocated(container, 3, 3)
- assert roleHistory.listOutstandingPlacedRequests().empty
+ assertOnContainerAllocationOpen(container, 3, 3)
+ assertNoOutstandingRequests()
}
}
[19/20] incubator-slider git commit: SLIDER-799 SLIDER-817 track
unplaced outstanding requests
Posted by st...@apache.org.
SLIDER-799 SLIDER-817 track unplaced outstanding requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43c61fbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43c61fbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43c61fbb
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 43c61fbba2ee448f2797790aaaf52ddaaf9ac6f5
Parents: 1938323
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 24 16:31:02 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 24 16:31:02 2015 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 5 +-
.../slider/server/appmaster/state/AppState.java | 1 +
.../appmaster/state/ContainerAllocation.java | 46 ++++++++++
.../appmaster/state/OutstandingRequest.java | 28 +++++-
.../state/OutstandingRequestTracker.java | 96 +++++++++++++++++---
.../server/appmaster/state/RoleHistory.java | 24 +++--
...tRoleHistoryOutstandingRequestTracker.groovy | 95 +++++++++++++++----
.../TestRoleHistoryRequestTracking.groovy | 32 ++++---
.../appmaster/model/mock/MockPriority.groovy | 43 +++++++++
9 files changed, 314 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index ce52b89..b6cb42b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -995,11 +995,12 @@ public final class SliderUtils {
return "null container";
}
return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s",
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
container.getId(),
container.getNodeId(),
container.getNodeHttpAddress(),
- container.getPriority());
+ container.getPriority(),
+ container.getResource());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 34b0492..20c1792 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1945,6 +1945,7 @@ public class AppState {
ContainerPriority.createPriority(role.getPriority(), true);
Priority p2 =
ContainerPriority.createPriority(role.getPriority(), false);
+ // TODO Delegate to Role History
operations.add(new CancelRequestOperation(p1, p2, toCancel));
role.cancel(toCancel);
excess -= toCancel;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
new file mode 100644
index 0000000..306ffb2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+/**
+ * This is just a tuple of the outcome of a container allocation
+ */
+public class ContainerAllocation {
+
+ /**
+ * What was the outcome of this allocation: placed, escalated, ...
+ */
+ public ContainerAllocationOutcome outcome;
+
+ /**
+ * The outstanding request which originated this.
+ * This will be null if the outcome is {@link ContainerAllocationOutcome#Unallocated}
+ * as it wasn't expected.
+ */
+ public OutstandingRequest origin;
+
+ public ContainerAllocation(ContainerAllocationOutcome outcome,
+ OutstandingRequest origin) {
+ this.outcome = outcome;
+ this.origin = origin;
+ }
+
+ public ContainerAllocation() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 8c320f0..24946af 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -91,6 +91,11 @@ public final class OutstandingRequest {
private boolean mayEscalate;
/**
+ * Priority of request; only valid after the request is built up
+ */
+ private int priority = -1;
+
+ /**
* Create a request
* @param roleId role
* @param node node -can be null
@@ -144,6 +149,10 @@ public final class OutstandingRequest {
return issuedRequest;
}
+ public int getPriority() {
+ return priority;
+ }
+
/**
* Build a container request.
* <p>
@@ -215,6 +224,7 @@ public final class OutstandingRequest {
mayEscalate = false;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
+ priority = pri.getPriority();
issuedRequest = new AMRMClient.ContainerRequest(resource,
hosts,
null,
@@ -236,6 +246,8 @@ public final class OutstandingRequest {
public synchronized AMRMClient.ContainerRequest escalate() {
Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued "+ this);
escalated = true;
+
+ // this is now the priority
Priority pri = ContainerPriority.createPriority(roleId, true);
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
@@ -258,10 +270,13 @@ public final class OutstandingRequest {
/**
* Mark the request as completed (or canceled).
+ * <p>
+ * Current action: if a node is defined, its request count is decremented
*/
public void completed() {
- assert node != null : "null node";
- node.getOrCreate(roleId).requestCompleted();
+ if (node != null) {
+ node.getOrCreate(roleId).requestCompleted();
+ }
}
/**
@@ -277,6 +292,15 @@ public final class OutstandingRequest {
}
/**
+ * Query for the resource requirements matching; always false before a request is issued
+ * @param resource
+ * @return
+ */
+ public synchronized boolean resourceRequirementsMatch(Resource resource) {
+ return issuedRequest != null && issuedRequest.getCapability().equals(resource);
+ }
+
+ /**
* Equality is on hostname and role
* @param o other
* @return true on a match
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index e226a22..05a8052 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -18,8 +18,12 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
@@ -33,6 +37,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
/**
@@ -61,9 +66,15 @@ public class OutstandingRequestTracker {
new HashMap<>();
/**
- * Create a new request for the specific role. If a
- * location is set, the request is added to the list of requests to track.
- * if it isn't, it is not tracked.
+ * List of open requests; no specific details on them.
+ */
+ private List<OutstandingRequest> openRequests = new ArrayList<>();
+
+ /**
+ * Create a new request for the specific role.
+ * <p>
+ * If a location is set, the request is added to {@link #placedRequests}.
+ * If not, it is added to {@link #openRequests}
* <p>
* This does not update the node instance's role's request count
* @param instance node instance to manager
@@ -75,6 +86,8 @@ public class OutstandingRequestTracker {
new OutstandingRequest(role, instance);
if (request.isLocated()) {
placedRequests.put(request, request);
+ } else {
+ openRequests.add(request);
}
return request;
}
@@ -85,7 +98,9 @@ public class OutstandingRequestTracker {
* @param hostname hostname
* @return the request or null if there was no outstanding one in the {@link #placedRequests}
*/
- public synchronized OutstandingRequest lookup(int role, String hostname) {
+ @VisibleForTesting
+ public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) {
+ Preconditions.checkArgument(hostname != null, "null hostname");
return placedRequests.get(new OutstandingRequest(role, hostname));
}
@@ -94,7 +109,8 @@ public class OutstandingRequestTracker {
* @param request matching request to find
* @return the request or null for no match in the {@link #placedRequests}
*/
- public synchronized OutstandingRequest remove(OutstandingRequest request) {
+ @VisibleForTesting
+ public synchronized OutstandingRequest removePlacedRequest(OutstandingRequest request) {
return placedRequests.remove(request);
}
@@ -103,22 +119,62 @@ public class OutstandingRequestTracker {
* from the {@link #placedRequests} structure.
* @param role role index
* @param hostname hostname
+ * @param resource
* @return the allocation outcome
*/
- public synchronized ContainerAllocationOutcome onContainerAllocated(int role, String hostname) {
+ public synchronized ContainerAllocationOutcome onContainerAllocated(int role,
+ String hostname,
+ Container container) {
+ ContainerAllocationOutcome outcome;
OutstandingRequest request =
placedRequests.remove(new OutstandingRequest(role, hostname));
- if (request == null) {
- // not in the list; this is an open placement
- return ContainerAllocationOutcome.Open;
- } else {
+ if (request != null) {
//satisfied request
+ log.info("Found placed request for container: {}", request);
request.completed();
// derive outcome from status of tracked request
- return request.isEscalated()
+ outcome = request.isEscalated()
? ContainerAllocationOutcome.Escalated
: ContainerAllocationOutcome.Placed;
+ } else {
+ // not in the list; this is an open placement
+ // scan through all containers in the open request list
+ request = removeOpenRequest(container);
+ if (request != null) {
+ log.info("Found open request for container: {}", request);
+ request.completed();
+ outcome = ContainerAllocationOutcome.Open;
+ } else {
+ log.warn("Container allocation was not expected :"
+ + SliderUtils.containerToString(container));
+ outcome = ContainerAllocationOutcome.Unallocated;
+ }
+ }
+ return outcome;
+ }
+
+ /**
+ * Find and remove an open request. Determine it by scanning open requests
+ * for one whose priority & resource requirements match that of the container
+ * allocated.
+ * @param container container allocated
+ * @return a request which matches the allocation, or null for "no match"
+ */
+ private OutstandingRequest removeOpenRequest(Container container) {
+ int pri = container.getPriority().getPriority();
+ Resource resource = container.getResource();
+ OutstandingRequest request = null;
+ ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
+ while (openlist.hasNext() && request == null) {
+ OutstandingRequest r = openlist.next();
+ if (r.getPriority() == pri
+ && r.resourceRequirementsMatch(resource)) {
+ // match of priority and resources
+ request = r;
+ openlist.remove();
+ }
}
+ return request;
}
/**
@@ -226,6 +282,13 @@ public class OutstandingRequestTracker {
hosts.add(request.node);
}
}
+ ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
+ while (openlist.hasNext()) {
+ OutstandingRequest next = openlist.next();
+ if (next.roleId == role) {
+ openlist.remove();
+ }
+ }
return hosts;
}
@@ -234,11 +297,20 @@ public class OutstandingRequestTracker {
* are shared
* @return a list of the current outstanding requests
*/
- public synchronized List<OutstandingRequest> listOutstandingRequests() {
+ public synchronized List<OutstandingRequest> listPlacedRequests() {
return new ArrayList<>(placedRequests.values());
}
/**
+ * Get a list of outstanding requests. The list is cloned, but the contents
+ * are shared
+ * @return a list of the current outstanding requests
+ */
+ public synchronized List<OutstandingRequest> listOpenRequests() {
+ return new ArrayList<>(openRequests);
+ }
+
+ /**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 64f9184..0b981b8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -643,12 +643,12 @@ public class RoleHistory {
int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
- LinkedList<NodeInstance> nodeInstances =
- getOrCreateNodesForRoleId(role);
- ContainerAllocationOutcome outcome = outstandingRequests.onContainerAllocated(role, hostname);
+ List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
+ ContainerAllocationOutcome outcome =
+ outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
- // tag nodes as available
+ // clear all the lists, so returning nodes to the available set
List<NodeInstance>
hosts = outstandingRequests.resetOutstandingRequests(role);
if (!hosts.isEmpty()) {
@@ -841,8 +841,18 @@ public class RoleHistory {
* @return a list of the requests outstanding at the time of requesting
*/
@VisibleForTesting
- public synchronized List<OutstandingRequest> listOutstandingPlacedRequests() {
- return outstandingRequests.listOutstandingRequests();
+ public List<OutstandingRequest> listPlacedRequests() {
+ return outstandingRequests.listPlacedRequests();
+ }
+
+
+ /**
+ * Get a snapshot of the outstanding placed request list
+ * @return a list of the requests outstanding at the time of requesting
+ */
+ @VisibleForTesting
+ public List<OutstandingRequest> listOpenRequests() {
+ return outstandingRequests.listOpenRequests();
}
/**
@@ -856,7 +866,6 @@ public class RoleHistory {
return lst;
}
-
/**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
@@ -864,6 +873,5 @@ public class RoleHistory {
public List<AbstractRMOperation> escalateOutstandingRequests() {
long now = now();
return outstandingRequests.escalateOutstandingRequests(now);
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index c30537a..3f41dfd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -23,11 +23,15 @@ import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockContainer
+import org.apache.slider.server.appmaster.model.mock.MockNodeId
+import org.apache.slider.server.appmaster.model.mock.MockPriority
import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
+import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -49,9 +53,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testAddRetrieveEntry() throws Throwable {
OutstandingRequest request = tracker.newRequest(host1, 0)
- assert tracker.lookup(0, "host1").equals(request)
- assert tracker.remove(request).equals(request)
- assert !tracker.lookup(0, "host1")
+ assert tracker.lookupPlacedRequest(0, "host1").equals(request)
+ assert tracker.removePlacedRequest(request).equals(request)
+ assert !tracker.lookupPlacedRequest(0, "host1")
}
@Test
@@ -59,11 +63,68 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
tracker.newRequest(host1, 0)
tracker.newRequest(host2, 0)
tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1") == ContainerAllocationOutcome.Placed
- assert !tracker.lookup(1, "host1")
- assert tracker.lookup(0, "host1")
+ assert tracker.onContainerAllocated(1, "host1", null) == ContainerAllocationOutcome.Placed
+ assert !tracker.lookupPlacedRequest(1, "host1")
+ assert tracker.lookupPlacedRequest(0, "host1")
}
-
+
+ @Test
+ public void testResetOpenRequests() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ assert !req1.located
+ tracker.newRequest(host1, 0)
+ def openRequests = tracker.listOpenRequests()
+ assert openRequests.size() == 1
+ tracker.resetOutstandingRequests(0)
+ assert tracker.listOpenRequests().empty
+ assert tracker.listPlacedRequests().empty
+ }
+
+ @Test
+ public void testRemoveOpenRequestUnissued() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ assert tracker.listOpenRequests().size() == 1
+ def c1 = factory.newContainer()
+ c1.setPriority(new MockPriority(0))
+
+ def resource = factory.newResource()
+ resource.virtualCores=1
+ resource.memory = 48;
+ c1.setResource(resource)
+ ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1)
+ assert outcome == ContainerAllocationOutcome.Unallocated
+ assert tracker.listOpenRequests().size() == 1
+ }
+
+ @Test
+ public void testIssuedOpenRequest() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ def resource = factory.newResource()
+ resource.virtualCores = 1
+ resource.memory = 48;
+ def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, "")
+ assert tracker.listOpenRequests().size() == 1
+ def c1 = factory.newContainer()
+
+ def nodeId = factory.newNodeId()
+ c1.nodeId = nodeId
+ nodeId.host ="hostname-1"
+
+ def pri = ContainerPriority.buildPriority(0, false)
+ assert pri > 0
+ c1.setPriority(new MockPriority(pri))
+
+ c1.setResource(resource)
+
+ def issued = req1.issuedRequest
+ assert issued.capability == resource
+ assert issued.priority.priority == c1.getPriority().getPriority()
+ assert req1.resourceRequirementsMatch(resource)
+ ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, nodeId.host, c1)
+ assert tracker.listOpenRequests().size() == 0
+ assert outcome == ContainerAllocationOutcome.Open
+ }
+
@Test
public void testResetEntries() throws Throwable {
tracker.newRequest(host1, 0)
@@ -73,14 +134,13 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert canceled.size() == 2
assert canceled.contains(host1)
assert canceled.contains(host2)
- assert tracker.lookup(1, "host1")
- assert !tracker.lookup(0, "host1")
+ assert tracker.lookupPlacedRequest(1, "host1")
+ assert !tracker.lookupPlacedRequest(0, "host1")
canceled = tracker.resetOutstandingRequests(0)
assert canceled.size() == 0
assert tracker.resetOutstandingRequests(1).size() == 1
}
-
@Test
public void testEscalation() throws Throwable {
@@ -92,7 +152,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert outstanding0.located
assert !outstanding0.escalated
assert !initialRequest.relaxLocality
- assert tracker.listOutstandingRequests().size() == 1
+ assert tracker.listPlacedRequests().size() == 1
// second. This one doesn't get launched. This is to verify that the escalation
// process skips entries which are in the list but have not been issued.
@@ -137,7 +197,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert escalations3.size() == 2
assert outstanding2.escalated
- // finally add a strict entry to th emix
+ // finally add a strict entry to the mix
def (res3, outstanding3) = newRequest(role1Status)
final ProviderRole providerRole1 = role1Status.providerRole
assert providerRole1.placementPolicy == PlacementPolicy.STRICT
@@ -156,7 +216,8 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
entry.containerCompleted(false)
entry.containerCompleted(false)
assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ final AMRMClient.ContainerRequest initialRequest =
+ outstanding0.buildContainerRequest(res0, role0Status, 0, null)
assert initialRequest.relaxLocality
assert initialRequest.nodes == null
}
@@ -171,16 +232,16 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
entry.containerCompleted(false)
entry.containerCompleted(false)
assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0,
- roleStatus, 0, null)
+ final AMRMClient.ContainerRequest initialRequest =
+ outstanding0.buildContainerRequest(res0, roleStatus, 0, null)
assert !initialRequest.relaxLocality
assert initialRequest.nodes[0] == host1.hostname
}
/**
* Create a new request (always against host1)
- * @param r
- * @return
+ * @param r role status
+ * @return (resource, oustanding-request)
*/
public def newRequest(RoleStatus r) {
final Resource res2 = new MockResource()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index bab6233..34a0a4d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.ProviderRole
@@ -123,7 +124,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
@@ -131,44 +132,44 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
assertOnContainerAllocated(container, 2, 1)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
- public void assertOnContainerAllocated(MockContainer c1, int p1, int p2) {
+ public void assertOnContainerAllocated(Container c1, int p1, int p2) {
assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2)
}
- public void assertOnContainerAllocationOpen(MockContainer c1, int p1, int p2) {
+ public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2)
}
- def assertNoOutstandingRequests() {
- assert roleHistory.listOutstandingPlacedRequests().empty
+ def assertNoOutstandingPlacedRequests() {
+ assert roleHistory.listPlacedRequests().empty
}
public void assertOutstandingPlacedRequests(int i) {
- assert roleHistory.listOutstandingPlacedRequests().size() == i
+ assert roleHistory.listPlacedRequests().size() == i
}
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assertOnContainerAllocated(container, 2, 1)
assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
assertOnContainerAllocated(container, 2, 2)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
@Test
@@ -176,7 +177,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assertOnContainerAllocated(container, 2, 1)
@@ -189,9 +190,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
- assertOnContainerAllocationOpen(container, 3, 3)
+ assert ContainerAllocationOutcome.Unallocated == roleHistory.onContainerAllocated(container, 3, 3)
// yet the list is now empty
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
+ roleHistory.listOpenRequests().empty
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -211,10 +213,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
assertOnContainerAllocated(container, 3, 2)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
container = factory.newContainer(req3, "three")
assertOnContainerAllocationOpen(container, 3, 3)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
new file mode 100644
index 0000000..9737973
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.Priority
+
+class MockPriority extends Priority {
+
+ private int priority;
+
+ MockPriority(int priority) {
+ this.priority = priority
+ }
+
+ MockPriority() {
+ }
+
+ @Override
+ int getPriority() {
+ return priority
+ }
+
+ @Override
+ void setPriority(int priority) {
+
+ }
+}
[08/20] incubator-slider git commit: SLIDER-799 escalation process
implemented; tests are TBD
Posted by st...@apache.org.
SLIDER-799 escalation process implemented; tests are TBD
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6d955af0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6d955af0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6d955af0
Branch: refs/heads/feature/SLIDER-799-AM-managed-relax
Commit: 6d955af0f4f6d02ec4f3080191a060464513f6b6
Parents: 559ed00
Author: Steve Loughran <st...@apache.org>
Authored: Mon Mar 16 20:08:51 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Mar 16 20:08:51 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 12 +++-
.../org/apache/slider/api/ResourceKeys.java | 2 +-
.../providers/AbstractProviderService.java | 5 ++
.../server/appmaster/SliderAppMaster.java | 47 +++++++++++++---
.../actions/EscalateOutstandingRequests.java | 22 ++++++++
.../operations/AsyncRMOperationHandler.java | 8 ++-
.../operations/CancelSingleRequest.java | 58 ++++++++++++++++++++
.../ProviderNotifyingOperationHandler.java | 5 ++
.../operations/RMOperationHandlerActions.java | 12 +++-
.../slider/server/appmaster/state/AppState.java | 33 ++++++++---
.../server/appmaster/state/NodeInstance.java | 29 ++++------
.../appmaster/state/OutstandingRequest.java | 11 +++-
.../state/OutstandingRequestTracker.java | 38 ++++++++++++-
.../server/appmaster/state/RoleHistory.java | 34 ++++++++----
.../history/TestRoleHistoryNIComparators.groovy | 1 -
.../model/mock/MockProviderService.groovy | 5 ++
.../model/mock/MockRMOperationHandler.groovy | 11 +++-
17 files changed, 280 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index b360fbe..38494a2 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -89,6 +89,7 @@ public interface InternalKeys {
* Default short life threshold: {@value}
*/
int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
+
/**
* Version of the app: {@value}
*/
@@ -177,5 +178,14 @@ public interface InternalKeys {
* 100% for chaos values
*/
int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
-
+
+ /**
+ * interval between checks for escalation: {@value}
+ */
+ String ESCALATION_CHECK_INTERVAL = "escalation.check.interval.seconds";
+
+ /**
+ * default value: {@value}
+ */
+ int DEFAULT_ESCALATION_CHECK_INTERVAL = 30;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index ce2a54f..94ce681 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -145,7 +145,7 @@ public interface ResourceKeys {
* Time in seconds to relax placement delay
*/
String PLACEMENT_RELAX_DELAY =
- "yarn.placement.relax.delay.seconds";
+ "yarn.placement.relax.seconds";
/**
* Time to have a strict placement policy outstanding before
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index fd7df73..7cba840 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -405,6 +405,11 @@ public abstract class AbstractProviderService
}
@Override
+ public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+ // no-op
+ }
+
+ @Override
public int cancelContainerRequests(Priority priority1,
Priority priority2,
int count) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 34bf20c..a93c60b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -113,6 +113,7 @@ import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
+import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.QueueService;
@@ -141,7 +142,6 @@ import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
@@ -918,7 +918,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
initCompleted.set(true);
scheduleFailureWindowResets(instanceDefinition.getResources());
-
+ scheduleEscalation(instanceDefinition.getInternal());
try {
@@ -1664,13 +1664,28 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info(
"Scheduling the failure window reset interval to every {} seconds",
seconds);
- RenewingAction<ResetFailureWindow> renew = new RenewingAction<ResetFailureWindow>(
+ RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
reset, seconds, seconds, TimeUnit.SECONDS, 0);
actionQueues.renewing("failures", renew);
} else {
log.info("Failure window reset interval is not set");
}
}
+
+ /**
+ * Schedule the escalation action
+ * @param internal
+ * @throws BadConfigException
+ */
+ private void scheduleEscalation(ConfTree internal) throws BadConfigException {
+ EscalateOutstandingRequests escalate = new EscalateOutstandingRequests();
+ ConfTreeOperations ops = new ConfTreeOperations(internal);
+ int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL,
+ InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL);
+ RenewingAction<EscalateOutstandingRequests> renew = new RenewingAction<>(
+ escalate, seconds, seconds, TimeUnit.SECONDS, 0);
+ actionQueues.renewing("escalation", renew);
+ }
/**
* Look at where the current node state is -and whether it should be changed
@@ -1728,13 +1743,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
queue(new ActionStopSlider(e));
}
}
-
+
+ /**
+ * Escalate operation as triggered by external timer.
+ * <p>
+ * Get the list of new operations off the AM, then executest them.
+ */
+ public void escalateOutstandingRequests() {
+ List<AbstractRMOperation> operations = appState.escalateOutstandingRequests();
+ providerRMOperationHandler.execute(operations);
+ execute(operations);
+ }
+
+
/**
* Shutdown operation: release all containers
*/
private void releaseAllContainers() {
+ List<AbstractRMOperation> operations = appState.releaseAllContainers();
+ providerRMOperationHandler.execute(operations);
//now apply the operations
- execute(appState.releaseAllContainers());
+ execute(operations);
}
/**
@@ -1808,8 +1837,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return rmOperationHandler.cancelContainerRequests(priority1, priority2, count);
}
-
-/* =================================================================== */
+ @Override
+ public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+ rmOperationHandler.cancelSingleRequest(request);
+ }
+
+ /* =================================================================== */
/* END */
/* =================================================================== */
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
new file mode 100644
index 0000000..e527be6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/EscalateOutstandingRequests.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+public class EscalateOutstandingRequests {
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
index 1cbb960..7c98551 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
@@ -78,7 +78,7 @@ public class AsyncRMOperationHandler extends RMOperationHandler {
break;
}
// a single release
- client.removeContainerRequest(request);
+ cancelSingleRequest(request);
remaining --;
}
}
@@ -86,6 +86,12 @@ public class AsyncRMOperationHandler extends RMOperationHandler {
}
@Override
+ public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+ // a single release
+ client.removeContainerRequest(request);
+ }
+
+ @Override
public void releaseAssignedContainer(ContainerId containerId) {
log.debug("Releasing container {}", containerId);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
new file mode 100644
index 0000000..e4ccb10
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+
+/**
+ * Cancel a container request
+ */
+public class CancelSingleRequest extends AbstractRMOperation {
+
+ private final Priority priority1;
+ private final Priority priority2;
+ private final int count;
+
+ public CancelSingleRequest(Priority priority1, Priority priority2, int count) {
+ this.priority1 = priority1;
+ this.priority2 = priority2;
+ this.count = count;
+ }
+
+ @Override
+ public void execute(RMOperationHandlerActions handler) {
+ handler.cancelContainerRequests(priority1, priority2, count);
+ }
+
+ @Override
+ public String toString() {
+ return "release " + count
+ + " requests for " + ContainerPriority.toString(priority1)
+ + " and " + ContainerPriority.toString(priority2);
+ }
+
+ /**
+ * Get the number to release
+ * @return the number of containers to release
+ */
+ public int getCount() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
index 66df566..184a36a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
@@ -47,4 +47,9 @@ public class ProviderNotifyingOperationHandler extends RMOperationHandler {
int count) {
return providerService.cancelContainerRequests(priority1, priority2, count);
}
+
+ @Override
+ public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+ providerService.cancelSingleRequest(request);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
index 97fde09..594ee47 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
@@ -27,9 +27,19 @@ import java.util.List;
public interface RMOperationHandlerActions {
void releaseAssignedContainer(ContainerId containerId);
- void addContainerRequest(AMRMClient.ContainerRequest req);
+ /**
+ * Issue a container request
+ * @param request
+ */
+ void addContainerRequest(AMRMClient.ContainerRequest request);
/**
+ * Cancel a specific request
+ * @param request request to cancel
+ */
+ void cancelSingleRequest(AMRMClient.ContainerRequest request);
+
+ /**
* Remove a container request
* @param priority1 priority to remove at
* @param priority2 second priority to target
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index f2c237c..a887107 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import static org.apache.slider.api.StateValues.*;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
@@ -88,8 +87,20 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.slider.api.ResourceKeys.*;
-import static org.apache.slider.api.RoleKeys.*;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_LABEL_EXPRESSION;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.YARN_LABEL_EXPRESSION;
+import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES;
+import static org.apache.slider.api.StateValues.STATE_CREATED;
+import static org.apache.slider.api.StateValues.STATE_DESTROYED;
+import static org.apache.slider.api.StateValues.STATE_LIVE;
+import static org.apache.slider.api.StateValues.STATE_SUBMITTED;
/**
@@ -704,7 +715,7 @@ public class AppState {
clusterStatusTemplate =
ClusterDescriptionOperations.buildFromInstanceDefinition(
- instanceDefinition);
+ instanceDefinition);
// Add the -site configuration properties
@@ -1286,7 +1297,7 @@ public class AppState {
int maxVal) {
String val = resources.getComponentOpt(name, option,
- Integer.toString(defVal));
+ Integer.toString(defVal));
Integer intVal;
if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
@@ -1856,11 +1867,19 @@ public class AppState {
}
roleHistory.resetFailedRecently();
}
+
+ /**
+ * Escalate operation as triggered by external timer.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public List<AbstractRMOperation> escalateOutstandingRequests() {
+ return roleHistory.escalateOutstandingRequests();
+ }
/**
* Look at the allocation status of one role, and trigger add/release
* actions if the number of desired role instances doesn't equal
- * (actual+pending).
+ * (actual + pending).
* <p>
* MUST be executed from within a synchronized method
* <p>
@@ -2046,7 +2065,7 @@ public class AppState {
Collection<RoleInstance> targets = cloneOwnedContainerList();
log.info("Releasing {} containers", targets.size());
List<AbstractRMOperation> operations =
- new ArrayList<AbstractRMOperation>(targets.size());
+ new ArrayList<>(targets.size());
for (RoleInstance instance : targets) {
if (instance.roleId == SliderKeys.ROLE_AM_PRIORITY_INDEX) {
// don't worry about the AM
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 71b74fc..68c8a15 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -41,7 +41,7 @@ public class NodeInstance {
*/
public NodeInstance(String hostname, int roles) {
this.hostname = hostname;
- nodeEntries = new ArrayList<NodeEntry>(roles);
+ nodeEntries = new ArrayList<>(roles);
}
/**
@@ -103,12 +103,8 @@ public class NodeInstance {
*/
public boolean isConsideredUnreliable(int role, int threshold) {
NodeEntry entry = get(role);
-
- if (entry != null) {
- return entry.getFailedRecently() > threshold;
- } else {
- return false;
- }
+
+ return entry != null && entry.getFailedRecently() > threshold;
}
/**
@@ -131,7 +127,8 @@ public class NodeInstance {
/**
- * run through each entry; gc'ing & removing old ones
+ * run through each entry; gc'ing & removing old ones that don't have
+ * a recent failure count (we care about those)
* @param absoluteTime age in millis
* @return true if there are still entries left
*/
@@ -140,7 +137,7 @@ public class NodeInstance {
ListIterator<NodeEntry> entries = nodeEntries.listIterator();
while (entries.hasNext()) {
NodeEntry entry = entries.next();
- if (entry.notUsedSince(absoluteTime)) {
+ if (entry.notUsedSince(absoluteTime) && entry.getFailedRecently() == 0) {
entries.remove();
} else {
active = true;
@@ -203,13 +200,8 @@ public class NodeInstance {
/**
* A comparator for sorting entries where the node is preferred over another.
- * If there's no entry for an element then it's failure count is set to -1, age to 0
- * for the purposes of the comparison
- * <ol>
- * <li>Entry exists => end of list as unknown</li>
- * <li>Lowest failure count</li>
- * <li>Age</li>
- * </ol>
+ * <p>
+ * The exact algorithm may change
*
* @return +ve int if left is preferred to right; -ve if right over left, 0 for equal
*/
@@ -227,7 +219,8 @@ public class NodeInstance {
NodeEntry left = o1.get(role);
NodeEntry right = o2.get(role);
- // sort by failure count first
+/*
+ // sort by failure count
int failL = left != null ? left.getFailedRecently() : -1;
int failR = right != null ? right.getFailedRecently() : -1;
@@ -237,7 +230,7 @@ public class NodeInstance {
if (failR > failL) {
return -1;
}
-
+ */
// failure counts are equal: compare age
long ageL = left != null ? left.getLastUsed() : 0;
long ageR = right != null ? right.getLastUsed() : 0;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index f6b83a7..7ad3fbb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -187,7 +187,7 @@ public final class OutstandingRequest {
* as the original one, and the same host, but: relaxed placement, and a changed priority
* so as to place it into the relaxed list.
*/
- public AMRMClient.ContainerRequest buildEscalatedContainerRequest() {
+ public AMRMClient.ContainerRequest escalate() {
escalated = true;
Preconditions.checkNotNull(issuedRequest, "issued request");
Priority pri = ContainerPriority.createPriority(roleId, true);
@@ -220,6 +220,15 @@ public final class OutstandingRequest {
}
/**
+ * Query to see if the request is ready to be escalated
+ * @param time time to check against
+ * @return true if escalation should begin
+ */
+ public boolean shouldEscalate(long time) {
+ return !escalated && escalationTimeoutMillis < time;
+ }
+
+ /**
* Equality is on hostname and role
* @param o other
* @return true on a match
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index e197a86..eb58e74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -19,6 +19,10 @@
package org.apache.slider.server.appmaster.state;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +52,11 @@ public class OutstandingRequestTracker {
protected static final Logger log =
LoggerFactory.getLogger(OutstandingRequestTracker.class);
+ /**
+ * no requests; saves creating a new list if not needed
+ */
+ private final List<AbstractRMOperation> NO_REQUESTS = new ArrayList<>(0);
+
private Map<OutstandingRequest, OutstandingRequest> placedRequests =
new HashMap<>();
@@ -122,10 +131,10 @@ public class OutstandingRequestTracker {
}
/**
- * Get the age of a container. If it is not known in the history,
+ * Get the age of a node hosting container. If it is not known in the history,
* return 0.
* @param c container
- * @return age, null if
+ * @return age, null if there's no entry for it.
*/
private long getAgeOf(Container c) {
long age = 0;
@@ -224,4 +233,29 @@ public class OutstandingRequestTracker {
public synchronized List<OutstandingRequest> listOutstandingRequests() {
return new ArrayList<>(placedRequests.values());
}
+
+ /**
+ * Escalate operation as triggered by external timer.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public synchronized List<AbstractRMOperation> escalateOutstandingRequests(long now) {
+ if (placedRequests.isEmpty()) {
+ return NO_REQUESTS;
+ }
+
+ List<AbstractRMOperation> operations = new ArrayList<>();
+ for (OutstandingRequest outstandingRequest : placedRequests.values()) {
+ if (outstandingRequest.shouldEscalate(now)) {
+
+ // time to escalate
+ CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.issuedRequest);
+ operations.add(cancel);
+ AMRMClient.ContainerRequest escalated =
+ outstandingRequest.escalate();
+ operations.add(new ContainerRequestOperation(escalated));
+ }
+
+ }
+ return operations;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index acfe606..0d4de2d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -18,17 +18,7 @@
package org.apache.slider.server.appmaster.state;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
@@ -38,12 +28,22 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* The Role History.
@@ -854,4 +854,14 @@ public class RoleHistory {
return lst;
}
+
+ /**
+ * Escalate operation as triggered by external timer.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public List<AbstractRMOperation> escalateOutstandingRequests() {
+ long now = now();
+ return outstandingRequests.escalateOutstandingRequests(now);
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
index 74dfd42..b26b2f0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -63,7 +63,6 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
def preferred = new NodeInstance.Preferred(0)
assert preferred.compare(age6failing, age1failing) == -1
assert preferred.compare(age1failing, age6failing) == 1
- assert preferred.compare(age1failing, age1failing) == 0
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index f8ab56d..44415f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -276,6 +276,11 @@ class MockProviderService implements ProviderService {
}
@Override
+ void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+
+ }
+
+ @Override
void execute(List<AbstractRMOperation> operations) {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6d955af0/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 297c597..a68ce02 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -39,7 +39,7 @@ class MockRMOperationHandler extends RMOperationHandler {
@Override
public void releaseAssignedContainer(ContainerId containerId) {
operations.add(new ContainerReleaseOperation(containerId))
- log.info("Releasing container ID " + containerId.getId())
+ log.info("Releasing container ID " + containerId.containerId)
releases++;
}
@@ -61,6 +61,15 @@ class MockRMOperationHandler extends RMOperationHandler {
return releaseable;
}
+ @Override
+ void cancelSingleRequest(AMRMClient.ContainerRequest request) {
+ // here assume that there is a copy of this request in the list
+ if (availableToCancel > 0) {
+ availableToCancel--;
+ cancelled++;
+ }
+ }
+
/**
* clear the history
*/