You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/21 19:58:45 UTC
[1/2] git commit: SLIDER-148 one final unit test on the exists command
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-460-stderr [created] a8b249bfa
SLIDER-148 one final unit test on the exists command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4dbc1161
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4dbc1161
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4dbc1161
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 4dbc11610262d8d506e8f13143186218bc077930
Parents: ac76781
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 18:57:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 21 18:57:00 2014 +0100
----------------------------------------------------------------------
.../groovy/org/apache/slider/agent/actions/TestActionExists.groovy | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4dbc1161/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 2645e27..277fa7d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -124,6 +124,7 @@ class TestActionExists extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
+ assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, args)
}
}
[2/2] git commit: SLIDER-460 move all tests from scanning stdout to
parsing the JSON from cluster status operations and examining it via the
ClusterDescription class
Posted by st...@apache.org.
SLIDER-460 move all tests from scanning stdout to parsing the JSON from cluster status operations and examining it via the ClusterDescription class
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a8b249bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a8b249bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a8b249bf
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: a8b249bfa381e9d31c19745a280e47ab7e82bdb1
Parents: 4dbc116
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 18:58:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 21 18:58:32 2014 +0100
----------------------------------------------------------------------
.../framework/AgentCommandTestBase.groovy | 66 +++++++++++++++-----
.../funtest/framework/CommandTestBase.groovy | 24 +++----
.../funtest/lifecycle/AMFailuresIT.groovy | 61 +++++++-----------
.../funtest/lifecycle/AgentFailures2IT.groovy | 37 +++--------
.../funtest/lifecycle/AgentFailuresIT.groovy | 41 +++---------
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 3 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 48 ++++++--------
7 files changed, 118 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 69523aa..effea44 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -128,25 +129,56 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
shell.dumpOutput();
}
- public static void assertComponentCount(String component, int count, SliderShell shell) {
- log.info("Asserting component count.")
- int instanceCount = getComponentCount(component, shell)
- assert count == instanceCount, 'Instance count for component did not match expected.'
- }
-
- public static int getComponentCount(String component, SliderShell shell) {
- String entry = findLineEntry(shell, ["instances", component] as String[])
- int instanceCount = 0
- if (!SliderUtils.isUnset(entry)) {
- log.info(entry)
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
- }
+
+ public ClusterDescription execStatus(String application) {
+ ClusterDescription cd
+ File statusFile = File.createTempFile("status", ".json")
+ try {
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ application,
+ ARG_OUTPUT, statusFile.absolutePath
+ ])
+
+ assert statusFile.exists()
+ cd = new ClusterDescription();
+ cd.fromFile(statusFile)
+ return cd
+ } finally {
+ statusFile.delete()
}
+ }
- return instanceCount
+ public int queryRequestedCount(String application, String role) {
+ ClusterDescription cd = execStatus(application)
+ int requestedCount = cd.statistics[role]["containers.requested"]
+ return requestedCount
+ }
+
+ boolean hasRequestedContainerCountExceeded(Map<String, String> args) {
+ String application = args['application']
+ String role = args['role']
+ int expectedCount = args['limit'].toInteger();
+ return queryRequestedCount(application, role) >= expectedCount
+ }
+
+ public ClusterDescription expectContainersLive(String clustername,
+ String component,
+ int count) {
+ ClusterDescription cd = execStatus(clustername)
+ assertContainersLive(cd, component, count)
+ return cd;
+ }
+
+ public static void assertContainersLive(ClusterDescription clusterDescription,
+ String component, int count) {
+ log.info("Asserting component count.")
+ int instanceCount = clusterDescription.instances[component].size()
+ if (count != instanceCount) {
+ log.warn(clusterDescription.toString())
+ }
+ assert count == instanceCount
}
public static String findLineEntry(SliderShell shell, String[] locaters) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 58c61ea..c68cac8 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -632,39 +632,29 @@ abstract class CommandTestBase extends SliderTestUtils {
])
+ sleep(5000)
+ ensureApplicationIsUp(cluster)
def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
DEFAULT_AM_RESTART_SLEEP_TIME)
sleep(sleeptime)
ClusterDescription status
- try {
- // am should have restarted it by now
- // cluster is live
- exists(0, cluster, true)
-
- status = sliderClient.clusterDescription
- } catch (SliderException e) {
- if (e.exitCode == EXIT_BAD_STATE) {
- log.error(
- "Property $YarnConfiguration.RM_AM_MAX_ATTEMPTS may be too low")
- }
- throw e;
- }
+ status = sliderClient.clusterDescription
return status
}
- protected void ensureApplicationIsUp(String clusterName) {
+ protected void ensureApplicationIsUp(String application) {
repeatUntilTrue(this.&isApplicationUp,
SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
1000,
- ['arg1': clusterName],
+ [application: application],
true,
- 'Application did not start, aborting test.')
+ 'Application did not start, failing test.')
}
protected boolean isApplicationUp(Map<String, String> args) {
- String applicationName = args['arg1'];
+ String applicationName = args['application'];
return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 159e2d6..323bf7d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -67,18 +67,27 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit : '1',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
+
// Wait for 20 secs for AM and agent to both reach STARTED state
sleep(1000 * 20)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
- assert live != null && live.isInteger() && live.toInteger() == 1,
- 'At least 1 container must be live now'
+ def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ def loggerInstances = cd.instances[COMMAND_LOGGER]
+ assert loggerInstances.size() == 1
+
+ def loggerStats = cd.statistics[COMMAND_LOGGER]
+
+ def origRequested = loggerStats["containers.requested"]
+ assert origRequested >= 2
+ assert loggerStats["containers.live"] == 1
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
assertSuccess(shell)
@@ -98,42 +107,16 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// Wait until AM comes back up and verify container count again
ensureApplicationIsUp(APPLICATION_NAME)
- // There should be exactly 1 live container
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
- assert live != null && live.isInteger() && live.toInteger() == 1,
- 'At least 1 container must be live now'
- log.info("After AM KILL: agent container is still live")
+ // There should be exactly 1 live logger container
+ def cd2 = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
// No new containers should be requested for the agents
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() == 0,
+ def loggerStats2 = cd2.statistics[COMMAND_LOGGER]
+ assert origRequested == loggerStats2["containers.requested"],
'No new agent containers should be requested'
- log.info("After AM KILL: no new agent containers were requested")
-
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
- assertSuccess(shell)
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
protected void killAMUsingAmSuicide() {
SliderShell shell = slider(EXIT_SUCCESS,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index a5cf5be..f1d5f84 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -59,41 +59,22 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 20, 1000 * 10, ['arg1': '3']);
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit: '3',
+ role: COMMAND_LOGGER ,
+ application:APPLICATION_NAME]);
sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell,
- ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
- 'At least 2 containers must be requested'
+ def cd = execStatus(APPLICATION_NAME)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
- assertSuccess(shell)
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a384170..97daff1 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -39,7 +39,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String APP_TEMPLATE2 =
"../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
-
@After
public void destroyCluster() {
cleanup(APPLICATION_NAME)
@@ -61,40 +60,20 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
-
- sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
+ repeatUntilTrue(this.&hasRequestedContainerCountExceeded,
+ 15,
+ 1000 * 10,
[
- ACTION_STATUS,
- APPLICATION_NAME])
+ application:APPLICATION_NAME,
+ role:COMMAND_LOGGER,
+ limit: '2'
+ ]);
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 2,
- 'At least 2 containers must be requested'
+ sleep(1000 * 20)
+ def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
-
- assertSuccess(shell)
}
-
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 9389f80..80b5db7 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -72,9 +72,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ACTION_STATUS,
APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
- assertSuccess(shell)
// get log folders
shell = slider(EXIT_SUCCESS,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a8b249bf/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index c87d507..1a5c5ff 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -88,13 +88,15 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 50,
+ 1000 * 10,
+ [limit : '1',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 1, shell)
+ expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
//flex
slider(EXIT_SUCCESS,
@@ -106,32 +108,22 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
"2"])
// sleep till the new instance starts
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
+ ensureApplicationIsUp(APPLICATION_NAME)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit : '3',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
+
+ sleep(1000 * 20)
+ def cd = execStatus(APPLICATION_NAME)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
- assertSuccess(shell)
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- int instanceCount = getComponentCount(COMMAND_LOGGER, shell)
- log.info("Noticed instance count - " + instanceCount + " for " + COMMAND_LOGGER)
- if (instanceCount >= expectedCount) {
- return true
- }
- return false
- }
}