You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/06/08 19:00:57 UTC
incubator-slider git commit: SLIDER-1138 pass memory and core reqs to
unique components
Repository: incubator-slider
Updated Branches:
refs/heads/develop baf6532dd -> c7a173cdc
SLIDER-1138 pass memory and core reqs to unique components
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c7a173cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c7a173cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c7a173cd
Branch: refs/heads/develop
Commit: c7a173cdcc15f675c1b4d471ff40eefb9afb1bae
Parents: baf6532
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 8 11:28:28 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 8 11:28:28 2016 -0700
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 26 +++++++++++---------
.../appstate/TestMockAppStateUniqueNames.groovy | 22 +++++++++++++++++
2 files changed, 36 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index ed87b89..b4465a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -802,6 +802,7 @@ public class AppState {
continue;
}
if (hasUniqueNames(resources, name)) {
+ // THIS NAME IS A GROUP
int desiredInstanceCount = getDesiredInstanceCount(resources, name);
Integer groupCount = 0;
if (groupCounts.containsKey(name)) {
@@ -853,21 +854,21 @@ public class AppState {
/**
* Get the desired instance count of a role, rejecting negative values
* @param resources resource map
- * @param role role name
+ * @param roleGroup role group
* @return the instance count
* @throws BadConfigException if the count is negative
*/
private int getDesiredInstanceCount(ConfTreeOperations resources,
- String role) throws BadConfigException {
+ String roleGroup) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
- log.error("Role {} has negative desired instances : {}", role,
+ log.error("Role {} has negative desired instances : {}", roleGroup,
desiredInstanceCount);
throw new BadConfigException(
"Negative instance count (%) requested for component %s",
- desiredInstanceCount, role);
+ desiredInstanceCount, roleGroup);
}
return desiredInstanceCount;
}
@@ -1353,12 +1354,12 @@ public class AppState {
* @throws NumberFormatException if the role could not be parsed.
*/
private int getResourceRequirement(ConfTreeOperations resources,
- String name,
+ String group,
String option,
int defVal,
int maxVal) {
- String val = resources.getComponentOpt(name, option,
+ String val = resources.getComponentOpt(group, option,
Integer.toString(defVal));
Integer intVal;
if (YARN_RESOURCE_MAX.equals(val)) {
@@ -1380,14 +1381,15 @@ public class AppState {
public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
+ String group = role.getGroup();
ConfTreeOperations resources = getResourcesSnapshot();
int cores = getResourceRequirement(resources,
- name,
+ group,
YARN_CORES,
DEF_YARN_CORES,
containerMaxCores);
capability.setVirtualCores(cores);
- int ram = getResourceRequirement(resources, name,
+ int ram = getResourceRequirement(resources, group,
YARN_MEMORY,
DEF_YARN_MEMORY,
containerMaxMemory);
@@ -1959,13 +1961,13 @@ public class AppState {
/**
* Get the node failure threshold for a specific role, falling back to
* the global one if not
- * @param roleName role name
+ * @param roleGroup role group
* @return the threshold for failures
*/
- private int getNodeFailureThresholdForRole(String roleName) {
+ private int getNodeFailureThresholdForRole(String roleGroup) {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- return resources.getComponentOptInt(roleName,
+ return resources.getComponentOptInt(roleGroup,
NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
index 5256163..d3ba798 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -71,6 +71,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "1024",
+ (ResourceKeys.YARN_CORES) : "2",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -90,6 +92,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "2",
(ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -103,6 +107,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group21").priority
assert 8 == appState.lookupRoleStatus("group22").priority
+ assert 1024 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group21").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group22").resourceRequirements.memory
+ assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores
}
@Test
@@ -111,6 +121,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "0",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -121,6 +133,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
appState.lookupRoleStatus(6)
assert 0 == appState.lookupRoleStatus("group11").desired
assert 6 == appState.lookupRoleStatus("group11").priority
+ assert 384 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 4 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
}
@Test
@@ -129,6 +143,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "3",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "640",
+ (ResourceKeys.YARN_CORES) : "8",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -145,6 +161,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group12").priority
assert 8 == appState.lookupRoleStatus("group13").priority
+ assert 640 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group12").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group13").resourceRequirements.memory
+ assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores
}
}