You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/07 00:29:55 UTC
[13/22] incubator-slider git commit: SLIDER-82: label information is
set in the ProviderRole,
no longer directly managed in AppState. Needed for building a sequence of AA
placement requests
SLIDER-82: label information is set in the ProviderRole, no longer directly managed in AppState. Needed for building a sequence of AA placement requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/07fb45df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/07fb45df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/07fb45df
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 07fb45dfacf5147fa3eba6ffade3f64ce4a69741
Parents: 138912c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 6 12:34:45 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 6 13:01:24 2015 +0000
----------------------------------------------------------------------
.../apache/slider/providers/ProviderRole.java | 25 +++--
.../slideram/SliderAMClientProvider.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 111 +++++++++----------
.../appmaster/state/OutstandingRequest.java | 15 ++-
.../server/appmaster/state/RoleHistory.java | 23 +---
.../server/appmaster/state/RoleStatus.java | 14 ++-
.../resources/AggregateConfResource.java | 26 ++---
.../appstate/TestMockAppStateAAPlacement.groovy | 2 +-
...tRoleHistoryOutstandingRequestTracker.groovy | 51 ++++++---
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../TestRoleHistoryRequestTracking.groovy | 10 +-
.../model/mock/BaseMockAppStateTest.groovy | 10 +-
.../appmaster/model/mock/MockFactory.groovy | 9 +-
13 files changed, 152 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 3009f50..1b95b42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -32,13 +32,15 @@ public final class ProviderRole {
public int placementPolicy;
public int nodeFailureThreshold;
public final long placementTimeoutSeconds;
+ public final String labelExpression;
public ProviderRole(String name, int id) {
this(name,
id,
PlacementPolicy.DEFAULT,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+ ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
}
/**
@@ -49,18 +51,20 @@ public final class ProviderRole {
* @param nodeFailureThreshold threshold for node failures (within a reset interval)
* after which a node failure is considered an app failure
* @param placementTimeoutSeconds for lax placement, timeout in seconds before
- * a relaxed placement request is generated.
+ * @param labelExpression label expression for requests; may be null
*/
public ProviderRole(String name,
int id,
int policy,
int nodeFailureThreshold,
- long placementTimeoutSeconds) {
+ long placementTimeoutSeconds,
+ String labelExpression) {
this.name = name;
this.id = id;
this.placementPolicy = policy;
this.nodeFailureThreshold = nodeFailureThreshold;
this.placementTimeoutSeconds = placementTimeoutSeconds;
+ this.labelExpression = labelExpression;
}
@Override
@@ -83,11 +87,14 @@ public final class ProviderRole {
@Override
public String toString() {
- return "ProviderRole {" +
- "name='" + name + '\'' +
- ", id=" + id +
- ", policy=" + placementPolicy +
- ", nodeFailureThreshold=" + nodeFailureThreshold +
- '}';
+ final StringBuilder sb = new StringBuilder("ProviderRole{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", id=").append(id);
+ sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold);
+ sb.append(", placementTimeoutSeconds=").append(placementTimeoutSeconds);
+ sb.append(", labelExpression='").append(labelExpression).append('\'');
+ sb.append('}');
+ return sb.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 9bd4dc9..3be0f48 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -90,7 +90,7 @@ public class SliderAMClientProvider extends AbstractClientProvider
new ProviderRole(COMPONENT_AM, KEY_AM,
PlacementPolicy.EXCLUDE_FROM_FLEXING,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- 0);
+ 0, "");
/**
* Initialize role list
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index e47ef34..946d45f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -22,7 +22,6 @@ import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -141,7 +140,7 @@ public class AppState {
private ConfTreeOperations resourcesSnapshot;
private ConfTreeOperations appConfSnapshot;
private ConfTreeOperations internalsSnapshot;
-
+
/**
* This is the status, the live model
*/
@@ -151,7 +150,7 @@ public class AppState {
* Metadata provided by the AM for use in filling in status requests
*/
private Map<String, String> applicationInfo;
-
+
/**
* Client properties created via the provider -static for the life
* of the application
@@ -278,10 +277,10 @@ public class AppState {
private RoleHistory roleHistory;
private Configuration publishedProviderConf;
private long startTimeThreshold;
-
+
private int failureThreshold = 10;
private int nodeFailureThreshold = 3;
-
+
private String logServerURL = "";
/**
@@ -618,7 +617,7 @@ public class AppState {
ResourceKeys.COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.DEFAULT));
int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
placementOpt, 0, 0, -1);
int placementTimeout =
component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY,
@@ -627,7 +626,8 @@ public class AppState {
priority,
placement,
getNodeFailureThresholdForRole(name),
- placementTimeout);
+ placementTimeout,
+ component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION));
log.info("New {} ", newRole);
return newRole;
}
@@ -653,7 +653,7 @@ public class AppState {
log.debug("Instance definition updated");
//note the time
snapshotTime = now();
-
+
// resolve references if not already done
instanceDefinition.resolve();
@@ -682,15 +682,14 @@ public class AppState {
clusterStatusTemplate =
ClusterDescriptionOperations.buildFromInstanceDefinition(
instanceDefinition);
-
// Add the -site configuration properties
for (Map.Entry<String, String> prop : clientProperties.entrySet()) {
clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue());
}
-
+
}
-
+
/**
* The resource configuration is updated -review and update state.
* @param resources updated resources specification
@@ -723,9 +722,9 @@ public class AppState {
private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
List<ProviderRole> newRoles = new ArrayList<>(0);
-
- //now update every role's desired count.
- //if there are no instance values, that role count goes to zero
+
+ // now update every role's desired count.
+ // if there are no instance values, that role count goes to zero
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
@@ -751,8 +750,8 @@ public class AppState {
}
}
- //now the dynamic ones. Iterate through the the cluster spec and
- //add any role status entries not in the role status
+ // now the dynamic ones. Iterate through the the cluster spec and
+ // add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
@@ -803,7 +802,7 @@ public class AppState {
* @throws BadConfigException if a role of that priority already exists
*/
public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException {
- //build role status map
+ // build role status map
int priority = providerRole.id;
if (roleStatusMap.containsKey(priority)) {
throw new BadConfigException("Duplicate Provider Key: %s and %s",
@@ -843,7 +842,7 @@ public class AppState {
//it is also added to the set of live nodes
getLiveContainers().put(containerId, am);
putOwnedContainer(containerId, am);
-
+
// patch up the role status
RoleStatus roleStatus = roleStatusMap.get(
(SliderKeys.ROLE_AM_PRIORITY_INDEX));
@@ -876,7 +875,12 @@ public class AppState {
return appMasterNode;
}
-
+ /**
+ * Look up the status entry of a role or raise an exception
+ * @param key role ID
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
public RoleStatus lookupRoleStatus(int key) {
RoleStatus rs = getRoleStatusMap().get(key);
if (rs == null) {
@@ -884,8 +888,15 @@ public class AppState {
}
return rs;
}
-
- public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
+
+ /**
+ * Look up the status entry of a container or raise an exception
+ *
+ * @param c container
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
+ public RoleStatus lookupRoleStatus(Container c) {
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
@@ -1093,8 +1104,7 @@ public class AppState {
}
return map;
}
-
-
+
/**
* Build a map of role->nodename->node-info
*
@@ -1173,10 +1183,10 @@ public class AppState {
RoleStatus role,
Resource capability) {
buildResourceRequirements(role, capability);
- String labelExpression = getLabelExpression(role);
+ String labelExpression = role.getLabelExpression();
//get the role history to select a suitable node, if available
AMRMClient.ContainerRequest containerRequest =
- createContainerRequest(role, capability, labelExpression);
+ createContainerRequest(role, capability);
return containerRequest;
}
@@ -1184,18 +1194,15 @@ public class AppState {
* Create a container request.
* Update internal state, such as the role request count
* This is where role history information will be used for placement decisions -
+ * @param labelExpression label expression to satisfy
* @param role role
* @param resource requirements
- * @param labelExpression label expression to satisfy
* @return the container request to submit
*/
private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
- Resource resource,
- String labelExpression) {
-
-
+ Resource resource) {
AMRMClient.ContainerRequest request;
- request = roleHistory.requestNode(role, resource, labelExpression);
+ request = roleHistory.requestNode(role, resource);
incrementRequestCount(role);
return request;
}
@@ -1211,7 +1218,6 @@ public class AppState {
incOutstandingContainerRequests();
}
-
/**
* dec requested count of a role
* <p>
@@ -1261,7 +1267,7 @@ public class AppState {
String option,
int defVal,
int maxVal) {
-
+
String val = resources.getComponentOpt(name, option,
Integer.toString(defVal));
Integer intVal;
@@ -1273,7 +1279,6 @@ public class AppState {
return intVal;
}
-
/**
* Build up the resource requirements for this role from the
* cluster specification, including substituing max allowed values
@@ -1299,17 +1304,6 @@ public class AppState {
}
/**
- * Extract the label expression for this role.
- * @param role role
- */
- public String getLabelExpression(RoleStatus role) {
- // Set up resource requirements from role values
- String name = role.getName();
- ConfTreeOperations resources = getResourcesSnapshot();
- return resources.getComponentOpt(name, YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION);
- }
-
- /**
* add a launched container to the node map for status responses
* @param container id
* @param node node details
@@ -1471,10 +1465,10 @@ public class AppState {
return sb.toString();
}
}
-
+
/**
* handle completed node in the CD -move something from the live
- * server list to the completed server list
+ * server list to the completed server list.
* @param status the node that has just completed
* @return NodeCompletionResult
*/
@@ -1534,7 +1528,7 @@ public class AppState {
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
message = String.format("Failure %s on host %s (%d): %s",
- roleInstance.getContainerId().toString(),
+ roleInstance.getContainerId(),
failedContainer.getNodeId().getHost(),
exitStatus,
completedLogsUrl);
@@ -1583,10 +1577,10 @@ public class AppState {
log.warn("Received notification of completion of unknown node {}", id);
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
-
+
// and the active node list if present
removeOwnedContainer(containerId);
-
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
containerId) : "container still in release queue";
@@ -1621,9 +1615,6 @@ public class AppState {
return completedLogsUrl;
}
-
-
-
/**
* Return the percentage done that Slider is to have YARN display in its
* Web UI
@@ -1652,7 +1643,7 @@ public class AppState {
public ClusterDescription refreshClusterStatus() {
return refreshClusterStatus(null);
}
-
+
/**
* Update the cluster description with the current application state
* @param providerStatus status from the provider for the cluster info section
@@ -1702,13 +1693,12 @@ public class AppState {
cd.statistics.put(rolename, stats);
}
-
Map<String, Integer> sliderstats = getLiveStatistics();
cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats);
-
+
// liveness
cd.liveness = getApplicationLivenessInformation();
-
+
return cd;
}
@@ -1723,7 +1713,7 @@ public class AppState {
li.allRequestsSatisfied = outstanding <= 0;
return li;
}
-
+
/**
* Get the live statistics map
* @return a map of statistics values, defined in the {@link StatusKeys}
@@ -1861,7 +1851,7 @@ public class AppState {
public List<AbstractRMOperation> escalateOutstandingRequests() {
return roleHistory.escalateOutstandingRequests();
}
-
+
/**
* Look at the allocation status of one role, and trigger add/release
* actions if the number of desired role instances doesn't equal
@@ -2103,7 +2093,6 @@ public class AppState {
//get the role
final ContainerId cid = container.getId();
final RoleStatus role = lookupRoleStatus(container);
-
//dec requested count
decrementRequestCount(role);
@@ -2213,7 +2202,7 @@ public class AppState {
cid,
roleName,
containerHostInfo);
-
+
//update app state internal structures and maps
RoleInstance instance = new RoleInstance(container);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 85bd259..38bc96f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -71,21 +71,21 @@ public final class OutstandingRequest {
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private AMRMClient.ContainerRequest issuedRequest;
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long requestedTimeMillis;
/**
* Time in millis after which escalation should be triggered..
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long escalationTimeoutMillis;
@@ -178,16 +178,15 @@ public final class OutstandingRequest {
* @param resource resource
* @param role role
* @param time time in millis to record as request time
- * @param labelExpression label to satisfy
* @return the request to raise
*/
public synchronized AMRMClient.ContainerRequest buildContainerRequest(
- Resource resource, RoleStatus role, long time, String labelExpression) {
+ Resource resource, RoleStatus role, long time) {
Preconditions.checkArgument(resource != null, "null `resource` arg");
Preconditions.checkArgument(role != null, "null `role` arg");
// cache label for escalation
- label = labelExpression;
+ label = role.getLabelExpression();
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
String[] hosts;
@@ -218,7 +217,7 @@ public final class OutstandingRequest {
escalated = true;
// and forbid it happening
mayEscalate = false;
- nodeLabels = labelExpression;
+ nodeLabels = label;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
priority = pri.getPriority();
@@ -254,7 +253,7 @@ public final class OutstandingRequest {
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
- if (label == null && issuedRequestNodes != null) {
+ if (SliderUtils.isUnset(label) && issuedRequestNodes != null) {
nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
} else {
nodes = null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index a0aa3bc..34340a2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -611,31 +611,14 @@ public class RoleHistory {
* Returns the request that is now being tracked.
* If the node instance is not null, it's details about the role is incremented
*
- *
* @param node node to target or null for "any"
* @param role role to request
- * @param labelExpression label to satisfy
* @return the container priority
*/
public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
- NodeInstance node, RoleStatus role, Resource resource, String labelExpression) {
+ NodeInstance node, RoleStatus role, Resource resource) {
OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
- return outstanding.buildContainerRequest(resource, role, now(), labelExpression);
- }
-
- /**
- * Find a node for a role and request an instance on that (or a location-less
- * instance) with a label expression
- * @param role role status
- * @param resource resource capabilities
- * @param labelExpression label to satisfy
- * @return a request ready to go
- */
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource,
- String labelExpression) {
- NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, labelExpression);
+ return outstanding.buildContainerRequest(resource, role, now());
}
/**
@@ -648,7 +631,7 @@ public class RoleHistory {
public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
Resource resource) {
NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, null);
+ return requestInstanceOnNode(node, role, resource);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 52df406..20f5802 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -41,7 +41,6 @@ public final class RoleStatus implements Cloneable {
* Role priority
*/
private final int key;
-
private final ProviderRole providerRole;
private int desired, actual, requested, releasing;
@@ -315,7 +314,7 @@ public final class RoleStatus implements Cloneable {
", actual=" + actual +
", requested=" + requested +
", releasing=" + releasing +
- ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount +
+ ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount +
", failed=" + failed +
", failed recently=" + failedRecently.get() +
", node failed=" + nodeFailed.get() +
@@ -324,6 +323,7 @@ public final class RoleStatus implements Cloneable {
", startFailed=" + startFailed +
", completed=" + completed +
", failureMessage='" + failureMessage + '\'' +
+ ", providerRole=" + providerRole +
'}';
}
@@ -373,7 +373,15 @@ public final class RoleStatus implements Cloneable {
info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount;
return info;
}
-
+
+ /**
+ * Get the (possibly null) label expression for this role
+ * @return a string or null
+ */
+ public String getLabelExpression() {
+ return providerRole.labelExpression;
+ }
+
/**
* Compare two role status entries by name
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
index 75d417b..ebffd1a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -37,23 +37,13 @@ public class AggregateConfResource {
public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) {
if (uriBuilder != null) {
- this.href =
- uriBuilder.build().toASCIIString();
- resources =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "resources"));
- internal =
- ResourceFactory.createConfTreeResource(conf.getInternal(),
- uriBuilder.clone().path(
- "configurations").path(
- "internal"));
- appConf =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "appConf"));
+ this.href = uriBuilder.build().toASCIIString();
+ resources = ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path("configurations").path("resources"));
+ internal = ResourceFactory.createConfTreeResource(conf.getInternal(),
+ uriBuilder.clone().path("configurations").path("internal"));
+ appConf = ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path("configurations").path("appConf"));
initConfMap();
} else {
resources = null;
@@ -63,7 +53,7 @@ public class AggregateConfResource {
}
private void initConfMap() {
- confMap = new HashMap<String, ConfTreeResource>();
+ confMap = new HashMap<>();
confMap.put("internal", internal);
confMap.put("resources", resources);
confMap.put("appConf", appConf);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 0e9fad0..6168146 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -44,7 +44,7 @@ import static org.apache.slider.server.appmaster.state.ContainerPriority.extract
class TestMockAppStateAAPlacement extends BaseMockAppStateTest
implements MockRoles {
- @Test
+// @Test
public void testAllocateAA() throws Throwable {
def aaRole = role2Status
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 8399d53..745d40f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -27,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -37,12 +38,28 @@ import org.junit.Test
class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+ public static final String WORKERS_LABEL = "workers"
NodeInstance host1 = new NodeInstance("host1", 3)
NodeInstance host2 = new NodeInstance("host2", 3)
def resource = factory.newResource(48, 1)
OutstandingRequestTracker tracker = new OutstandingRequestTracker()
+ public static final ProviderRole WORKER = new ProviderRole(
+ "worker",
+ 5,
+ PlacementPolicy.NONE,
+ 2,
+ 1,
+ WORKERS_LABEL)
+
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [ WORKER ] + bindingInfo.roles
+ bindingInfo
+ }
+
@Test
public void testAddRetrieveEntry() throws Throwable {
OutstandingRequest request = tracker.newRequest(host1, 0)
@@ -54,10 +71,10 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testAddCompleteEntry() throws Throwable {
def req1 = tracker.newRequest(host1, 0)
- req1.buildContainerRequest(resource, role0Status, 0, "")
+ req1.buildContainerRequest(resource, role0Status, 0)
- tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0, "")
- tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0, "")
+ tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0)
+ tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0)
def allocation = tracker.onContainerAllocated(1, "host1", null)
assert allocation.outcome == ContainerAllocationOutcome.Placed
@@ -82,7 +99,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testRemoveOpenRequestUnissued() throws Throwable {
def req1 = tracker.newRequest(null, 0)
- req1.buildContainerRequest(resource, role0Status, 0, "")
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
def c1 = factory.newContainer(null, new MockPriority(0))
c1.resource = resource
@@ -97,7 +114,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testIssuedOpenRequest() throws Throwable {
def req1 = tracker.newRequest(null, 0)
- req1.buildContainerRequest(resource, role0Status, 0, "")
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
def pri = ContainerPriority.buildPriority(0, false)
@@ -142,7 +159,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
// first request: default placement
assert role0Status.placementPolicy == PlacementPolicy.DEFAULT;
final def (res0, outstanding0) = newRequest(role0Status)
- final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0)
assert outstanding0.issuedRequest != null;
assert outstanding0.located
assert !outstanding0.escalated
@@ -178,7 +195,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
// build that second request from an anti-affine entry
// these get placed as well
now += interval
- final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now, null)
+ final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now)
// escalate a little bit more
final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now)
// and expect no new entries
@@ -219,18 +236,18 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
resource.virtualCores = 1
resource.memory = 48;
- def label = "workers"
+ def workerRole = lookupRole(WORKER.name)
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
+ def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == label)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
assert (yarnRequest2.relaxLocality)
}
/**
- * If the placement doesnt include a lablel, then the escalation request
+ * If the placement doesnt include a label, then the escalation request
* retains the node list, but sets relaxLocality==true
* @throws Throwable
*/
@@ -244,13 +261,13 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
def label = null
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
- assert (yarnRequest.nodes != null)
- assert (yarnRequest.nodeLabelExpression == null)
- assert (!yarnRequest.relaxLocality)
+ def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0)
+ assert yarnRequest.nodes != null
+ assert !yarnRequest.nodeLabelExpression
+ assert !yarnRequest.relaxLocality
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodes != null)
- assert (yarnRequest2.relaxLocality)
+ assert yarnRequest2.nodes != null
+ assert yarnRequest2.relaxLocality
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index a1e424f..254c0b6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -46,7 +47,8 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
3,
PlacementPolicy.STRICT,
3,
- 3)
+ 3,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
@Override
String getTestName() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 42d0c50..c6dcb07 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -86,8 +86,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
assertListEquals([age2Active0], roleHistory.cloneRecentNodeList(0))
roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
+ resource
+ )
}
@Test
@@ -106,8 +106,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// which is translated to a no-location request
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
+ resource
+ )
assertNull(req.nodes)
@@ -120,7 +120,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// looking for a node should now find one
ni = roleHistory.findNodeForNewInstance(roleStatus)
assert ni == age3Active0
- req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource, "")
+ req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource)
assert 1 == req.nodes.size()
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index a065518..40d7fd7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -128,15 +128,19 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
}
public RoleStatus getRole0Status() {
- return appState.lookupRoleStatus(ROLE0)
+ lookupRole(ROLE0)
+ }
+
+ public RoleStatus lookupRole(String role) {
+ appState.lookupRoleStatus(role)
}
public RoleStatus getRole1Status() {
- return appState.lookupRoleStatus(ROLE1)
+ lookupRole(ROLE1)
}
public RoleStatus getRole2Status() {
- return appState.lookupRoleStatus(ROLE2)
+ lookupRole(ROLE2)
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index f7fd641..0a4a93e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -49,21 +49,24 @@ class MockFactory implements MockRoles {
0,
PlacementPolicy.DEFAULT,
2,
- 1)
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
// role 1 is strict. timeout should be irrelevant; same as failures
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
1,
PlacementPolicy.STRICT,
2,
- 1)
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
// role 2: longer delay and anti-affinity
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
PlacementPolicy.ANTI_AFFINITY_REQUIRED,
2,
- 2)
+ 2,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
int appIdCount;
int attemptIdCount;