You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:15 UTC
[33/50] [abbrv] hadoop git commit: YARN-6405. Improve configuring
services through REST API. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 43c7ead..9f7b4a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -20,7 +20,13 @@ package org.apache.slider.server.appmaster.state;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.yarn.api.records.Container;
@@ -42,6 +48,7 @@ import org.apache.slider.api.proto.Messages.ComponentCountProto;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ApplicationState;
import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.RoleStatistics;
@@ -79,6 +86,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.api.ResourceKeys.*;
@@ -99,7 +107,6 @@ public class AppState {
private final AbstractClusterServices recordFactory;
private final MetricsAndMonitoring metricsAndMonitoring;
-
/**
* Flag set to indicate the application is live -this only happens
* after the buildInstance operation
@@ -108,9 +115,11 @@ public class AppState {
private Application app;
+ // priority_id -> RoleStatus
private final Map<Integer, RoleStatus> roleStatusMap =
new ConcurrentSkipListMap<>();
+ // component_name -> ProviderRole
private final Map<String, ProviderRole> roles =
new ConcurrentHashMap<>();
@@ -202,6 +211,10 @@ public class AppState {
private SliderMetrics appMetrics;
private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ // A cache for loading config files from remote such as hdfs
+ public LoadingCache<ConfigFile, Object> configFileCache = null;
+
/**
* Create an instance
* @param recordFactory factory for YARN records
@@ -304,8 +317,6 @@ public class AppState {
public synchronized void buildInstance(AppStateBindingInfo binding)
throws BadClusterStateException, BadConfigException, IOException {
binding.validate();
-
- log.debug("Building application state");
containerReleaseSelector = binding.releaseSelector;
// set the cluster specification (once its dependency the client properties
@@ -313,10 +324,8 @@ public class AppState {
this.app = binding.application;
appMetrics = SliderMetrics.register(app.getName(),
"Metrics for service");
- appMetrics
- .tag("type", "Metrics type [component or service]", "service");
- appMetrics
- .tag("appId", "Application id for service", app.getId());
+ appMetrics.tag("type", "Metrics type [component or service]", "service");
+ appMetrics.tag("appId", "Application id for service", app.getId());
org.apache.slider.api.resource.Configuration conf = app.getConfiguration();
startTimeThreshold =
@@ -327,12 +336,7 @@ public class AppState {
nodeFailureThreshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
DEFAULT_NODE_FAILURE_THRESHOLD);
- //build the initial role list
- List<ProviderRole> roleList = new ArrayList<>(binding.roles);
- for (ProviderRole providerRole : roleList) {
- buildRole(providerRole);
- }
-
+ //build the initial component list
int priority = 1;
for (Component component : app.getComponents()) {
priority = getNewPriority(priority);
@@ -340,25 +344,18 @@ public class AppState {
if (roles.containsKey(name)) {
continue;
}
- if (component.getUniqueComponentSupport()) {
- log.info("Skipping group " + name + ", as it's unique component");
- continue;
- }
log.info("Adding component: " + name);
- ProviderRole dynamicRole =
- createComponent(name, name, component, priority);
- buildRole(dynamicRole);
- roleList.add(dynamicRole);
+ createComponent(name, name, component, priority++);
}
+
//then pick up the requirements
- buildRoleRequirementsFromResources();
+// buildRoleRequirementsFromResources();
// set up the role history
roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
roleHistory.onStart(binding.fs, binding.historyPath);
// trigger first node update
roleHistory.onNodesUpdated(binding.nodeReports);
-
//rebuild any live containers
rebuildModelFromRestart(binding.liveContainers);
@@ -367,9 +364,39 @@ public class AppState {
//mark as live
applicationLive = true;
app.setState(STARTED);
+ createConfigFileCache(binding.fs);
+ }
+
+ private void createConfigFileCache(final FileSystem fileSystem) {
+ this.configFileCache =
+ CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
+ .build(new CacheLoader<ConfigFile, Object>() {
+ @Override public Object load(ConfigFile key) throws Exception {
+ switch (key.getType()) {
+ case HADOOP_XML:
+ try (FSDataInputStream input = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ org.apache.hadoop.conf.Configuration confRead =
+ new org.apache.hadoop.conf.Configuration(false);
+ confRead.addResource(input);
+ Map<String, String> map = new HashMap<>(confRead.size());
+ for (Map.Entry<String, String> entry : confRead) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+ return map;
+ }
+ case TEMPLATE:
+ try (FSDataInputStream fileInput = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ return IOUtils.toString(fileInput);
+ }
+ default:
+ return null;
+ }
+ }
+ });
}
- //TODO WHY do we need to create the component for AM ?
public ProviderRole createComponent(String name, String group,
Component component, int priority) throws BadConfigException {
org.apache.slider.api.resource.Configuration conf =
@@ -384,26 +411,28 @@ public class AppState {
DEF_YARN_LABEL_EXPRESSION);
ProviderRole newRole =
new ProviderRole(name, group, priority, (int)placementPolicy, threshold,
- placementTimeout, label, component);
-
+ placementTimeout, label, component, this);
+ buildRole(newRole, component);
log.info("Created a new role " + newRole);
return newRole;
}
@VisibleForTesting
- public synchronized List<ProviderRole> updateComponents(Map<String, Long>
+ public synchronized void updateComponents(Map<String, Long>
componentCounts) throws BadConfigException {
for (Component component : app.getComponents()) {
if (componentCounts.containsKey(component.getName())) {
- component.setNumberOfContainers(componentCounts.get(component
- .getName()));
+ long count = componentCounts.get(component.getName());
+ component.setNumberOfContainers(count);
+ ProviderRole role = roles.get(component.getName());
+ if (role != null && roleStatusMap.get(role.id) != null) {
+ setDesiredContainers(roleStatusMap.get(role.id), (int) count);
+ }
}
}
- //TODO update cluster description
- return buildRoleRequirementsFromResources();
}
- public synchronized List<ProviderRole> updateComponents(
+ public synchronized void updateComponents(
Messages.FlexComponentsRequestProto requestProto)
throws BadConfigException {
Map<String, Long> componentCounts = new HashMap<>();
@@ -412,116 +441,119 @@ public class AppState {
componentCounts.put(componentCount.getName(), componentCount
.getNumberOfContainers());
}
- return updateComponents(componentCounts);
+ updateComponents(componentCounts);
}
/**
* build the role requirements from the cluster specification
* @return a list of any dynamically added provider roles
*/
- private List<ProviderRole> buildRoleRequirementsFromResources()
- throws BadConfigException {
-
- List<ProviderRole> newRoles = new ArrayList<>(0);
-
- // now update every role's desired count.
- // if there are no instance values, that role count goes to zero
- // Add all the existing roles
- // component name -> number of containers
- Map<String, Integer> groupCounts = new HashMap<>();
-
- for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- if (roleStatus.isExcludeFromFlexing()) {
- // skip inflexible roles, e.g AM itself
- continue;
- }
- long currentDesired = roleStatus.getDesired();
- String role = roleStatus.getName();
- String roleGroup = roleStatus.getGroup();
- Component component = roleStatus.getProviderRole().component;
- int desiredInstanceCount = component.getNumberOfContainers().intValue();
-
- int newDesired = desiredInstanceCount;
- if (component.getUniqueComponentSupport()) {
- Integer groupCount = 0;
- if (groupCounts.containsKey(roleGroup)) {
- groupCount = groupCounts.get(roleGroup);
- }
-
- newDesired = desiredInstanceCount - groupCount;
-
- if (newDesired > 0) {
- newDesired = 1;
- groupCounts.put(roleGroup, groupCount + newDesired);
- } else {
- newDesired = 0;
- }
- }
-
- if (newDesired == 0) {
- log.info("Role {} has 0 instances specified", role);
- }
- if (currentDesired != newDesired) {
- log.info("Role {} flexed from {} to {}", role, currentDesired,
- newDesired);
- setDesiredContainers(roleStatus, newDesired);
- }
- }
-
- // now the dynamic ones. Iterate through the the cluster spec and
- // add any role status entries not in the role status
-
- for (Component component : app.getComponents()) {
- String name = component.getName();
- if (roles.containsKey(name)) {
- continue;
- }
- if (component.getUniqueComponentSupport()) {
- // THIS NAME IS A GROUP
- int desiredInstanceCount = component.getNumberOfContainers().intValue();
- Integer groupCount = 0;
- if (groupCounts.containsKey(name)) {
- groupCount = groupCounts.get(name);
- }
- for (int i = groupCount + 1; i <= desiredInstanceCount; i++) {
- // this is a new instance of an existing group
- String newName = String.format("%s%d", name, i);
- if (roles.containsKey(newName)) {
- continue;
- }
- int newPriority = getNewPriority(i);
- log.info("Adding new role {}", newName);
- ProviderRole dynamicRole =
- createComponent(newName, name, component, newPriority);
- RoleStatus newRole = buildRole(dynamicRole);
- incDesiredContainers(newRole);
- log.info("New role {}", newRole);
- if (roleHistory != null) {
- roleHistory.addNewRole(newRole);
- }
- newRoles.add(dynamicRole);
- }
- } else {
- // this is a new value
- log.info("Adding new role {}, num containers {}", name,
- component.getNumberOfContainers());
- ProviderRole dynamicRole =
- createComponent(name, name, component, getNewPriority(1));
- RoleStatus newRole = buildRole(dynamicRole);
- incDesiredContainers(newRole,
- component.getNumberOfContainers().intValue());
- log.info("New role {}", newRole);
- if (roleHistory != null) {
- roleHistory.addNewRole(newRole);
- }
- newRoles.add(dynamicRole);
- }
- }
- // and fill in all those roles with their requirements
- buildRoleResourceRequirements();
- return newRoles;
- }
+// private List<ProviderRole> buildRoleRequirementsFromResources()
+// throws BadConfigException {
+//
+// List<ProviderRole> newRoles = new ArrayList<>(0);
+//
+// // now update every role's desired count.
+// // if there are no instance values, that role count goes to zero
+// // Add all the existing roles
+// // component name -> number of containers
+// Map<String, Integer> groupCounts = new HashMap<>();
+//
+// for (RoleStatus roleStatus : getRoleStatusMap().values()) {
+// if (roleStatus.isExcludeFromFlexing()) {
+// // skip inflexible roles, e.g AM itself
+// continue;
+// }
+// long currentDesired = roleStatus.getDesired();
+// String role = roleStatus.getName();
+// String roleGroup = roleStatus.getGroup();
+// Component component = roleStatus.getProviderRole().component;
+// int desiredInstanceCount = component.getNumberOfContainers().intValue();
+//
+// int newDesired = desiredInstanceCount;
+// if (component.getUniqueComponentSupport()) {
+// Integer groupCount = 0;
+// if (groupCounts.containsKey(roleGroup)) {
+// groupCount = groupCounts.get(roleGroup);
+// }
+//
+// newDesired = desiredInstanceCount - groupCount;
+//
+// if (newDesired > 0) {
+// newDesired = 1;
+// groupCounts.put(roleGroup, groupCount + newDesired);
+// } else {
+// newDesired = 0;
+// }
+// }
+//
+// if (newDesired == 0) {
+// log.info("Role {} has 0 instances specified", role);
+// }
+// if (currentDesired != newDesired) {
+// log.info("Role {} flexed from {} to {}", role, currentDesired,
+// newDesired);
+// setDesiredContainers(roleStatus, newDesired);
+// }
+// }
+//
+// log.info("Counts per component: " + groupCounts);
+// // now the dynamic ones. Iterate through the the cluster spec and
+// // add any role status entries not in the role status
+//
+// List<RoleStatus> list = new ArrayList<>(getRoleStatusMap().values());
+// for (RoleStatus roleStatus : list) {
+// String name = roleStatus.getName();
+// Component component = roleStatus.getProviderRole().component;
+// if (roles.containsKey(name)) {
+// continue;
+// }
+// if (component.getUniqueComponentSupport()) {
+// // THIS NAME IS A GROUP
+// int desiredInstanceCount = component.getNumberOfContainers().intValue();
+// Integer groupCount = 0;
+// if (groupCounts.containsKey(name)) {
+// groupCount = groupCounts.get(name);
+// }
+// log.info("Component " + component.getName() + ", current count = "
+// + groupCount + ", desired count = " + desiredInstanceCount);
+// for (int i = groupCount + 1; i <= desiredInstanceCount; i++) {
+// int priority = roleStatus.getPriority();
+// // this is a new instance of an existing group
+// String newName = String.format("%s%d", name, i);
+// int newPriority = getNewPriority(priority + i - 1);
+// log.info("Adding new role {}", newName);
+// ProviderRole dynamicRole =
+// createComponent(newName, name, component, newPriority);
+// RoleStatus newRole = buildRole(dynamicRole);
+// incDesiredContainers(newRole);
+// log.info("New role {}", newRole);
+// if (roleHistory != null) {
+// roleHistory.addNewRole(newRole);
+// }
+// newRoles.add(dynamicRole);
+// }
+// } else {
+// // this is a new value
+// log.info("Adding new role {}", name);
+// ProviderRole dynamicRole =
+// createComponent(name, name, component, roleStatus.getPriority());
+// RoleStatus newRole = buildRole(dynamicRole);
+// incDesiredContainers(roleStatus,
+// component.getNumberOfContainers().intValue());
+// log.info("New role {}", newRole);
+// if (roleHistory != null) {
+// roleHistory.addNewRole(newRole);
+// }
+// newRoles.add(dynamicRole);
+// }
+// }
+// // and fill in all those roles with their requirements
+// buildRoleResourceRequirements();
+//
+// return newRoles;
+// }
private int getNewPriority(int start) {
if (!rolePriorityMap.containsKey(start)) {
@@ -539,16 +571,20 @@ public class AppState {
* @return the role status built up
* @throws BadConfigException if a role of that priority already exists
*/
- public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException {
+ public RoleStatus buildRole(ProviderRole providerRole, Component component)
+ throws BadConfigException {
// build role status map
int priority = providerRole.id;
if (roleStatusMap.containsKey(priority)) {
- throw new BadConfigException("Duplicate Provider Key: %s and %s",
- providerRole,
- roleStatusMap.get(priority)
- .getProviderRole());
+ throw new BadConfigException("Duplicate component priority Key: %s and %s",
+ providerRole, roleStatusMap.get(priority));
}
RoleStatus roleStatus = new RoleStatus(providerRole);
+ roleStatus.setResourceRequirements(buildResourceRequirements(roleStatus));
+ long prev = roleStatus.getDesired();
+ setDesiredContainers(roleStatus, component.getNumberOfContainers().intValue());
+ log.info("Set desired containers for component " + component.getName() +
+ " from " + prev + " to " + roleStatus.getDesired());
roleStatusMap.put(priority, roleStatus);
String name = providerRole.name;
roles.put(name, providerRole);
@@ -559,16 +595,6 @@ public class AppState {
}
/**
- * Build up the requirements of every resource
- */
- private void buildRoleResourceRequirements() {
- for (RoleStatus role : roleStatusMap.values()) {
- role.setResourceRequirements(buildResourceRequirements(role));
- log.info("Setting resource requirements for {} to {}", role.getName(),
- role.getResourceRequirements());
- }
- }
- /**
* Look up the status entry of a role or raise an exception
* @param key role ID
* @return the status entry
@@ -731,7 +757,7 @@ public class AppState {
}
/**
- * Enum all nodes by role.
+ * Enum all nodes by role.
* @param role role, or "" for all roles
* @return a list of nodes, may be empty
*/
@@ -785,7 +811,7 @@ public class AppState {
}
/**
- * Build a map of role->nodename->node-info
+ * Build a map of Component_name -> ContainerId -> ClusterNode
*
* @return the map of Role name to list of Cluster Nodes
*/
@@ -850,7 +876,7 @@ public class AppState {
/**
* Create a container request.
- * Update internal state, such as the role request count.
+ * Update internal state, such as the role request count.
* Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
* This is where role history information will be used for placement decisions.
* @param role role
@@ -942,18 +968,9 @@ public class AppState {
}
private void setDesiredContainers(RoleStatus role, int n) {
+ int delta = n - role.getComponentMetrics().containersDesired.value();
role.getComponentMetrics().containersDesired.set(n);
- appMetrics.containersDesired.set(n);
- }
-
- private void incDesiredContainers(RoleStatus role) {
- role.getComponentMetrics().containersDesired.incr();
- appMetrics.containersDesired.incr();
- }
-
- private void incDesiredContainers(RoleStatus role, int n) {
- role.getComponentMetrics().containersDesired.incr(n);
- appMetrics.containersDesired.incr(n);
+ appMetrics.containersDesired.incr(delta);
}
private void incCompletedContainers(RoleStatus role) {
@@ -1001,7 +1018,8 @@ public class AppState {
* Build up the resource requirements for this role from the cluster
* specification, including substituting max allowed values if the
* specification asked for it (except when
- * {@link ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} is set to false).
+ * {@link org.apache.slider.api.ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED}
+ * is set to false).
* @param role role
* during normalization
*/
@@ -1009,11 +1027,6 @@ public class AppState {
// Set up resource requirements from role values
String name = role.getName();
Component component = role.getProviderRole().component;
- if (component == null) {
- // this is for AM container
- // TODO why do we need to create the component for AM ?
- return Resource.newInstance(1, 512);
- }
int cores = DEF_YARN_CORES;
if (component.getResource() != null && component.getResource().getCpus()
!= null) {
@@ -1282,10 +1295,13 @@ public class AppState {
if (roleInstance != null) {
int roleId = roleInstance.roleId;
String rolename = roleInstance.role;
- log.info("Failed container in role[{}] : {}", roleId, rolename);
+ log.info("Failed container in role[{}] : {}", roleId,
+ roleInstance.getCompInstanceName());
try {
RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId);
decRunningContainers(roleStatus);
+ roleStatus.getProviderRole().failedInstanceName
+ .offer(roleInstance.compInstanceName);
boolean shortLived = isShortLived(roleInstance);
String message;
Container failedContainer = roleInstance.container;
@@ -1571,7 +1587,7 @@ public class AppState {
/**
* Look at the allocation status of one role, and trigger add/release
- * actions if the number of desired role instances doesn't equal
+ * actions if the number of desired role instances doesn't equal
* (actual + pending).
* <p>
* MUST be executed from within a synchronized method
@@ -1584,7 +1600,6 @@ public class AppState {
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
- log.info("review one role " + role.getName());
List<AbstractRMOperation> operations = new ArrayList<>();
long delta;
long expected;
@@ -1594,9 +1609,7 @@ public class AppState {
expected = role.getDesired();
}
- log.info("Reviewing {} : ", role);
- log.debug("Expected {}, Requested/Running {}, Delta: {}", expected,
- role.getActualAndRequested(), delta);
+ log.info("Reviewing " + role.getName() + ": " + role.getComponentMetrics());
checkFailureThreshold(role);
if (expected < 0 ) {
@@ -1729,7 +1742,9 @@ public class AppState {
for (RoleInstance possible : finalCandidates) {
log.info("Targeting for release: {}", possible);
containerReleaseSubmitted(possible.container);
- operations.add(new ContainerReleaseOperation(possible.getId()));
+ role.getProviderRole().failedInstanceName
+ .offer(possible.compInstanceName);
+ operations.add(new ContainerReleaseOperation(possible.getContainerId()));
}
}
@@ -1783,7 +1798,7 @@ public class AppState {
for (RoleInstance role : activeRoleInstances) {
if (role.container.getId().equals(containerId)) {
containerReleaseSubmitted(role.container);
- operations.add(new ContainerReleaseOperation(role.getId()));
+ operations.add(new ContainerReleaseOperation(role.getContainerId()));
}
}
@@ -1907,17 +1922,6 @@ public class AppState {
}
/**
- * Get diagnostics info about containers
- */
- public String getContainerDiagnosticInfo() {
- StringBuilder builder = new StringBuilder();
- for (RoleStatus roleStatus : getRoleStatusMap().values()) {
- builder.append(roleStatus).append('\n');
- }
- return builder.toString();
- }
-
- /**
* Event handler for the list of active containers on restart.
* Sets the info key {@link StatusKeys#INFO_CONTAINERS_AM_RESTART}
* to the size of the list passed down (and does not set it if none were)
@@ -1965,10 +1969,10 @@ public class AppState {
//update app state internal structures and maps
+ //TODO recover the component instance name from zk registry ?
RoleInstance instance = new RoleInstance(container);
instance.command = roleName;
instance.role = roleName;
- instance.group = role.getGroup();
instance.roleId = roleId;
instance.environment = new String[0];
instance.container = container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index de52f4e..736dfd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
@@ -27,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderRole;
@@ -42,6 +44,8 @@ public final class RoleInstance implements Cloneable {
public Container container;
public ProviderRole providerRole;
+ public long componentId = -1;
+ public String compInstanceName = null;
/**
* Container ID
*/
@@ -58,7 +62,6 @@ public final class RoleInstance implements Cloneable {
* Name of the role
*/
public String role;
- public String group;
/**
* Version of the app
@@ -106,7 +109,7 @@ public final class RoleInstance implements Cloneable {
public String host;
public String hostURL;
public ContainerAllocationOutcome placement;
-
+ public Path compInstanceDir;
/**
* A list of registered endpoints.
@@ -114,10 +117,24 @@ public final class RoleInstance implements Cloneable {
private List<Endpoint> endpoints =
new ArrayList<>(2);
- public RoleInstance(ContainerAssignment assignment) {
- this(assignment.container);
- placement = assignment.placement;
+ public RoleInstance(Container container, ProviderRole role) {
+ this(container);
+ if (role.componentIdCounter != null) {
+ componentId = role.componentIdCounter.getAndIncrement();
+ compInstanceName = role.name + componentId;
+ } else {
+ compInstanceName = role.name;
+ }
+ this.providerRole = role;
+ }
+
+ public RoleInstance(Container container, ProviderRole role,
+ String compInstanceName) {
+ this(container);
+ this.compInstanceName = compInstanceName;
+ this.providerRole = role;
}
+
/**
* Create an instance to track an allocated container
* @param container a container which must be non null, and have a non-null Id field.
@@ -136,10 +153,6 @@ public final class RoleInstance implements Cloneable {
hostURL = "http://" + container.getNodeHttpAddress();
}
}
-
- public ContainerId getId() {
- return container.getId();
- }
public NodeId getHost() {
return container.getNodeId();
@@ -151,6 +164,7 @@ public final class RoleInstance implements Cloneable {
new StringBuilder("RoleInstance{");
sb.append("role='").append(role).append('\'');
sb.append(", id='").append(id).append('\'');
+ sb.append(", instanceName='").append(compInstanceName).append('\'');
sb.append(", container=").append(SliderUtils.containerToString(container));
sb.append(", createTime=").append(createTime);
sb.append(", startTime=").append(startTime);
@@ -170,7 +184,7 @@ public final class RoleInstance implements Cloneable {
}
public ContainerId getContainerId() {
- return container != null ? container.getId() : null;
+ return container.getId();
}
/**
@@ -322,4 +336,8 @@ public final class RoleInstance implements Cloneable {
}
return info;
}
+
+ public String getCompInstanceName() {
+ return compInstanceName;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 5051aee..9842481 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -272,6 +272,7 @@ public final class RoleStatus implements MetricSet {
// containers -- maybe we need releasing
//if we are releasing, remove the number that are already released.
//but never switch to a positive
+ // TODO, WHY is this min operation even needed ??? if delta is negative, it's always < 0 ???
delta = Math.min(delta, 0);
}
return delta;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 118ca9d..5bc6dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -214,6 +214,7 @@ public interface StateAccessForProviders {
/**
* Find out about the nodes for specific roles
+ * Component_name -> ContainerId -> ClusterNode
* @return
*/
Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
index 0f6247d..ac89ed8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -59,7 +59,7 @@ public interface RestApiErrorMessages {
ERROR_RESOURCE_CPUS_INVALID_RANGE
+ " for component %s (or at the global level)";
String ERROR_CONTAINERS_COUNT_INVALID =
- "Required no of containers not specified";
+ "Invalid no of containers specified";
String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
index 776ce00..d7c72a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -20,17 +20,30 @@ package org.apache.slider.util;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.Configuration;
import org.apache.slider.api.resource.Resource;
import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class ServiceApiUtil {
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+public class ServiceApiUtil {
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceApiUtil.class);
@VisibleForTesting
- public static void validateApplicationPostPayload(Application application) {
+ public static void validateApplicationPayload(Application application,
+ FileSystem fs) throws IOException {
if (StringUtils.isEmpty(application.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
@@ -64,11 +77,13 @@ public class ServiceApiUtil {
application.getArtifact().getType());
// container size
- if (application.getNumberOfContainers() == null) {
+ if (application.getNumberOfContainers() == null
+ || application.getNumberOfContainers() < 0) {
throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID);
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": "
+ + application.getNumberOfContainers());
}
-
+ validateConfigFile(application.getConfiguration().getFiles(), fs);
// Since it is a simple app with no components, create a default component
application.getComponents().add(createDefaultComponent(application));
} else {
@@ -114,11 +129,13 @@ public class ServiceApiUtil {
if (comp.getNumberOfContainers() == null) {
comp.setNumberOfContainers(globalNumberOfContainers);
}
- if (comp.getNumberOfContainers() == null) {
+ if (comp.getNumberOfContainers() == null
+ || comp.getNumberOfContainers() < 0) {
throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID,
- comp.getName()));
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
+ + ": " + comp.getNumberOfContainers(), comp.getName()));
}
+ validateConfigFile(comp.getConfiguration().getFiles(), fs);
}
}
@@ -128,6 +145,46 @@ public class ServiceApiUtil {
}
}
+ // 1) Verify the src_file exists and non-empty for template
+ // 2) dest_file is absolute path
+ private static void validateConfigFile(List<ConfigFile> list, FileSystem fs)
+ throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : list) {
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+ // validate dest_file is absolute
+ if (!Paths.get(file.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must be absolute path: " + file.getDestFile());
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+ }
+ }
+
+
private static void validateApplicationResource(Resource resource,
Component comp, Artifact.TypeEnum artifactType) {
// Only apps/components of type APPLICATION can skip resource requirement
@@ -200,4 +257,8 @@ public class ServiceApiUtil {
comp.setLaunchCommand(app.getLaunchCommand());
return comp;
}
+
+ public static String $(String s) {
+ return "${" + s +"}";
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
index da2ed0d..9e79821 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java
@@ -67,7 +67,7 @@ public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
return new MockYarnEngine(8, 1);
}
- @Test
+ // TODO does not support adding new components dynamically
public void testDynamicRoleHistory() throws Throwable {
String dynamic = "dynamicRole";
@@ -81,12 +81,8 @@ public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
.COMPONENT_PLACEMENT_POLICY, "" + placementPolicy);
application.getComponents().add(component);
- // write the definitions
- List<ProviderRole> updates = appState.updateComponents(
+ appState.updateComponents(
Collections.singletonMap(dynamic, desired));
- assertEquals(1, updates.size());
- ProviderRole updatedRole = updates.get(0);
- assertEquals(updatedRole.placementPolicy, placementPolicy);
// now look at the role map
assertNotNull(appState.getRoleMap().get(dynamic));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
index 01bf9bd..6d8e963 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
@@ -87,7 +87,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
createAndStartNodes();
}
- @Test
+ // TODO does not support adding new components dynamically
public void testDynamicFlexAddRole() throws Throwable {
Application application = appState.getClusterStatus();
Component component = new Component().name("dynamicAdd7")
@@ -96,16 +96,12 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
appState.updateComponents(Collections.singletonMap(component.getName(),
component.getNumberOfContainers()));
createAndStartNodes();
- dumpClusterDescription("updated CD", appState.getClusterStatus());
appState.lookupRoleStatus("dynamicAdd7");
}
@Test
public void testDynamicFlexDropRole() throws Throwable {
appState.updateComponents(Collections.singletonMap("dynamic-6", 0L));
-
- Application getCD = appState.getClusterStatus();
- dumpClusterDescription("updated CD", getCD);
//status is retained for future
appState.lookupRoleStatus("dynamic-6");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index eaf5271..54ffe17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -26,10 +26,15 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles;
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
+import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.junit.Test;
import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
/**
* Test that if you have more than one role, the right roles are chosen for
@@ -72,40 +77,76 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
return application;
}
+ public static Map<String, RoleInstance> organize(List<RoleInstance>
+ instances) {
+ Map<String, RoleInstance> map = new TreeMap<>();
+ for (RoleInstance instance : instances) {
+ assertFalse("Multiple role instances for unique name " + instance
+ .compInstanceName, map.containsKey(instance.compInstanceName));
+ System.out.println("Adding to map " + instance.compInstanceName + " for" +
+ instance.role);
+ map.put(instance.compInstanceName, instance);
+ }
+ return map;
+ }
+
+ public static void verifyInstances(List<RoleInstance> instances, String
+ group, String... roles) {
+ assertEquals(roles.length, instances.size());
+ Map<String, RoleInstance> map = organize(instances);
+ int i = 0;
+ for (Entry<String, RoleInstance> entry : map.entrySet()) {
+ assertEquals(roles[i], entry.getKey());
+ RoleInstance instance = entry.getValue();
+ assertEquals(roles[i], instance.compInstanceName);
+ assertEquals(group, instance.role);
+ assertEquals(group, instance.providerRole.name);
+ assertEquals(group, instance.providerRole.group);
+ // TODO remove group from provider role if it continues to be unused
+ i++;
+ }
+ }
+
@Test
public void testDynamicFlexDown() throws Throwable {
createAndStartNodes();
+ List<RoleInstance> instances = appState.cloneOwnedContainerList();
+ verifyInstances(instances, "group1", "group10", "group11");
+
appState.updateComponents(Collections.singletonMap("group1", 0L));
createAndStartNodes();
- RoleStatus roleStatus = appState.lookupRoleStatus("group11");
+ instances = appState.cloneOwnedContainerList();
+ assertEquals(0, instances.size());
+
+ RoleStatus roleStatus = appState.lookupRoleStatus("group1");
assertEquals(0, roleStatus.getDesired());
assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize());
assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores());
assertEquals("group1", roleStatus.getGroup());
+
+ // now flex back up
+ appState.updateComponents(Collections.singletonMap("group1", 3L));
+ createAndStartNodes();
+ instances = appState.cloneOwnedContainerList();
+ verifyInstances(instances, "group1", "group10", "group11", "group12");
+ // fails because the names continue at N+1, with group12, group13, group14
}
@Test
public void testDynamicFlexUp() throws Throwable {
createAndStartNodes();
+ List<RoleInstance> instances = appState.cloneOwnedContainerList();
+ verifyInstances(instances, "group1", "group10", "group11");
+
appState.updateComponents(Collections.singletonMap("group1", 3L));
createAndStartNodes();
- RoleStatus group11 = appState.lookupRoleStatus("group11");
- RoleStatus group12 = appState.lookupRoleStatus("group12");
- RoleStatus group13 = appState.lookupRoleStatus("group13");
- assertEquals(1, group11.getDesired());
- assertEquals(1, group12.getDesired());
- assertEquals(1, group13.getDesired());
- assertEquals(1024L, group11.getResourceRequirements().getMemorySize());
- assertEquals(1024L, group12.getResourceRequirements().getMemorySize());
- assertEquals(1024L, group13.getResourceRequirements().getMemorySize());
- assertEquals(2, group11.getResourceRequirements().getVirtualCores());
- assertEquals(2, group12.getResourceRequirements().getVirtualCores());
- assertEquals(2, group13.getResourceRequirements().getVirtualCores());
- assertEquals("group1", group11.getGroup());
- assertEquals("group1", group12.getGroup());
- assertEquals("group1", group13.getGroup());
-
- appState.refreshClusterStatus();
+ instances = appState.cloneOwnedContainerList();
+ verifyInstances(instances, "group1", "group10", "group11", "group12");
+
+ RoleStatus group1 = appState.lookupRoleStatus("group1");
+ assertEquals(3, group1.getDesired());
+ assertEquals(1024L, group1.getResourceRequirements().getMemorySize());
+ assertEquals("group1", group1.getGroup());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
index 046bd83..d382c8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
@@ -27,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockAppState;
import org.apache.slider.server.appmaster.model.mock.MockRoles;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.junit.Test;
import java.util.Collections;
@@ -47,6 +48,11 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
role0.resource(new org.apache.slider.api.resource.Resource().memory("512")
.cpus(2));
+ // hack - because role0 is created before the test run
+ RoleStatus role0Status =
+ appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id);
+ role0Status.setResourceRequirements(
+ appState.buildResourceRequirements(role0Status));
appState.updateComponents(Collections.singletonMap(role0.getName(),
role0.getNumberOfContainers()));
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
@@ -58,12 +64,17 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
assertEquals(2, requirements.getVirtualCores());
}
+ //TODO replace with resource profile feature in yarn
@Test
public void testMaxMemAllocations() throws Throwable {
// max core allocations no longer supported
Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0);
role0.resource(new org.apache.slider.api.resource.Resource()
.memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2));
+ RoleStatus role0Status =
+ appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id);
+ role0Status.setResourceRequirements(
+ appState.buildResourceRequirements(role0Status));
appState.updateComponents(Collections.singletonMap(role0.getName(),
role0.getNumberOfContainers()));
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
index eca8401..4352959 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -176,7 +176,14 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
*/
public RoleInstance roleInstance(ContainerAssignment assigned) {
Container target = assigned.container;
- RoleInstance ri = new RoleInstance(target);
+ String failedInstance =
+ assigned.role.getProviderRole().failedInstanceName.poll();
+ RoleInstance ri;
+ if (failedInstance != null) {
+ ri = new RoleInstance(target, assigned.role.getProviderRole(), failedInstance);
+ } else {
+ ri = new RoleInstance(target, assigned.role.getProviderRole());
+ }
ri.roleId = assigned.role.getPriority();
ri.role = assigned.role.getName();
return ri;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ace79d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java
index 112a5ac..4098cf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java
@@ -30,6 +30,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
@@ -118,7 +119,8 @@ public class MockProviderService implements ProviderService {
@Override
public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
Application application, Container container, ProviderRole providerRole,
- SliderFileSystem sliderFileSystem) throws IOException, SliderException {
+ SliderFileSystem sliderFileSystem, RoleInstance roleInstance)
+ throws IOException, SliderException {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org