You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/06/15 15:10:01 UTC
[1/2] incubator-slider git commit: SLIDER-1142 increase AM memory for
some funtests and add checks for None in agent
Repository: incubator-slider
Updated Branches:
refs/heads/develop 24a5062ac -> 2b41da2d5
SLIDER-1142 increase AM memory for some funtests and add checks for None in agent
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2b41da2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2b41da2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2b41da2d
Branch: refs/heads/develop
Commit: 2b41da2d56a6165eb54fa0fc9cab6900fad9d177
Parents: 454f2cf
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 15 07:21:15 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 15 08:08:37 2016 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/agent/ActionQueue.py | 2 +-
slider-agent/src/main/python/agent/main.py | 2 +-
slider-core/src/test/app_packages/test_command_log/resources.json | 2 +-
.../test/app_packages/test_command_log/resources_add_on_pkg.json | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index be6af53..7514337 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -164,7 +164,7 @@ class ActionQueue(threading.Thread):
if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
- if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
+ if 'roleParams' in command and command['roleParams'] is not None and ActionQueue.AUTO_RESTART in command['roleParams']:
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
if store_command:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index 39de1ae..1932a37 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -60,7 +60,7 @@ def signal_handler(signum, frame):
if docker_mode:
tmpdir = controller.actionQueue.dockerManager.stop_container()
- if controller is not None and hasattr(controller, 'stopCommand'):
+ if controller is not None and hasattr(controller, 'stopCommand') and controller.stopCommand is not None:
controller.stopCommand = _increment_task_id(controller.stopCommand)
controller.appGracefulStopQueued = True
controller.actionQueue.execute_command(controller.stopCommand)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-core/src/test/app_packages/test_command_log/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources.json b/slider-core/src/test/app_packages/test_command_log/resources.json
index 18c505f..f6a401a 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources.json
@@ -11,7 +11,7 @@
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
index 3f71eb3..25e090e 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
@@ -11,7 +11,7 @@
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
[2/2] incubator-slider git commit: SLIDER-1141 add status for unique
components
Posted by bi...@apache.org.
SLIDER-1141 add status for unique components
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/454f2cf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/454f2cf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/454f2cf2
Branch: refs/heads/develop
Commit: 454f2cf2e391bc261f7905181a310994976973cf
Parents: 24a5062
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 15 07:18:42 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 15 08:08:37 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/api/RoleKeys.java | 5 ++++
.../slider/server/appmaster/state/AppState.java | 14 ++++++++--
.../appstate/TestMockAppStateUniqueNames.groovy | 29 ++++++++++++++++++++
3 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 647ac23..812a6b3 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -30,6 +30,11 @@ public interface RoleKeys {
String ROLE_NAME = "role.name";
/**
+ * The group of a role: {@value}
+ */
+ String ROLE_GROUP = "role.group";
+
+ /**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index b4465a8..3213d93 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -42,7 +42,6 @@ import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
@@ -1346,7 +1345,7 @@ public class AppState {
* These are returned as integers, but there is special handling of the
* string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
* the return of the maximum value.
- * @param name component to get from
+ * @param group component to get from
* @param option option name
* @param defVal default value
* @param maxVal value to return if the max val is requested
@@ -1800,11 +1799,20 @@ public class AppState {
for (RoleStatus role : getRoleStatusMap().values()) {
String rolename = role.getName();
+ if (hasUniqueNames(instanceDefinition.getResourceOperations(),
+ role.getGroup())) {
+ cd.setRoleOpt(rolename, COMPONENT_PRIORITY, role.getPriority());
+ cd.setRoleOpt(rolename, ROLE_GROUP, role.getGroup());
+ MapOperations groupOptions = instanceDefinition.getResourceOperations()
+ .getComponent(role.getGroup());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename),
+ groupOptions.options);
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
- cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
+ cd.setRoleOpt(rolename, ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
cd.setRoleOpt(rolename, ROLE_RELEASING_INSTANCES, role.getReleasing());
cd.setRoleOpt(rolename, ROLE_FAILED_INSTANCES, role.getFailed());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
index d3ba798..84d6bc7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
@@ -86,6 +87,25 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
return new ConfTreeOperations(resources)
}
+ private static void checkRole(MockAppState appState,
+ String roleName,
+ String roleGroup,
+ Map<String, String> expectedOpts) {
+
+ for (String key : expectedOpts.keySet()) {
+ if (ResourceKeys.COMPONENT_PRIORITY.equals(key) ||
+ ResourceKeys.COMPONENT_INSTANCES.equals(key)) {
+ continue
+ }
+ assert expectedOpts.get(key).equals(appState.getClusterStatus()
+ .getMandatoryRoleOpt(roleName, key))
+ }
+ assert 1 == appState.getClusterStatus().getMandatoryRoleOptInt(
+ roleName, ResourceKeys.COMPONENT_INSTANCES)
+ assert roleGroup.equals(appState.getClusterStatus().getMandatoryRoleOpt(
+ roleName, RoleKeys.ROLE_GROUP))
+ }
+
@Test
public void testDynamicFlexAddRole() throws Throwable {
def cd = init()
@@ -113,6 +133,10 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores
assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group21", "group2", opts)
+ checkRole(appState, "group22", "group2", opts)
}
@Test
@@ -167,6 +191,11 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores
assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group11", "group1", opts)
+ checkRole(appState, "group12", "group1", opts)
+ checkRole(appState, "group13", "group1", opts)
}
}