You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/23 12:40:40 UTC
[01/10] git commit: SLIDER-460 unused imports
Repository: incubator-slider
Updated Branches:
refs/heads/develop ec07bdc12 -> 5d9a053b2
SLIDER-460 unused imports
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1f84c6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1f84c6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1f84c6b
Branch: refs/heads/develop
Commit: e1f84c6b0697972c21b928d8404f43708e7e674b
Parents: fe5ded1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 22 21:32:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:54 2014 +0100
----------------------------------------------------------------------
.../slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1f84c6b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 3ab7897..0a5163a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -20,13 +20,10 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
[08/10] git commit: SLIDER-460 debug-level logging of exists actions
Posted by st...@apache.org.
SLIDER-460 debug-level logging of exists actions
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/75030d23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/75030d23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/75030d23
Branch: refs/heads/develop
Commit: 75030d231896be7e8d0b24ad6f79052dbfe6bf53
Parents: e1f84c6
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 10:39:35 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:55 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 ++++--
.../slider/client/SliderYarnClientImpl.java | 21 +++++++++++++++++---
2 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/75030d23/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 6ea305e..38ba827 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1740,7 +1740,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
verifyBindingsDefined();
SliderUtils.validateClusterName(name);
boolean checkLive = args.live;
- log.debug("actionExists({}, {})", name, checkLive);
+ log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
//initial probe for a cluster in the filesystem
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
@@ -1764,6 +1764,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (checkLive) {
// the app exists, check that it is not in any terminated state
YarnApplicationState appstate = instance.getYarnApplicationState();
+ log.debug(" current app state = {}", appstate);
inDesiredState =
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
} else {
@@ -1773,7 +1774,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
YarnApplicationState desiredState =
YarnApplicationState.valueOf(state);
ApplicationReport foundInstance =
- yarnClient.findClusterInInstanceList(userInstances, name, desiredState);
+ yarnClient.findAppInInstanceList(userInstances, name, desiredState);
if (foundInstance != null) {
// found in selected state: success
inDesiredState = true;
@@ -1791,6 +1792,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.debug("State {}", report);
return EXIT_FALSE;
} else {
+ log.debug("Application instance is in desired state");
log.info("Application {} is {}\n{}", name,
instance.getYarnApplicationState(), report);
return EXIT_SUCCESS;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/75030d23/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index c793bd1..a00f3b2 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -274,19 +274,34 @@ public class SliderYarnClientImpl extends YarnClientImpl {
return found;
}
- public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
- String appname,
+ /**
+ * Find an app in the instance list in the desired state
+ * @param instances instance list
+ * @param appname application name
+ * @param desiredState yarn state desired
+ * @return the match or null for none
+ */
+ public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances,
+ String appname,
YarnApplicationState desiredState) {
ApplicationReport found = null;
ApplicationReport foundAndLive = null;
+ log.debug("Searching {} records for instance name {} in state '{}'",
+ instances.size(), appname, desiredState);
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)) {
- if (app.getYarnApplicationState().equals(desiredState)) {
+
+ YarnApplicationState appstate =
+ app.getYarnApplicationState();
+ log.debug("app ID {} is in state {}", app.getApplicationId(), appstate);
+ if (appstate.equals(desiredState)) {
+ log.debug("match");
return app;
}
}
}
// nothing found in desired state
+ log.debug("No match");
return null;
}
[09/10] git commit: Merge branch 'feature/SLIDER-460-stderr' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-460-stderr' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c87b082d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c87b082d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c87b082d
Branch: refs/heads/develop
Commit: c87b082d7c087d14879a94676106bfaeb23e2b02
Parents: ec07bdc 802ff46
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 11:36:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:36:46 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 61 ++++---
.../slider/client/SliderYarnClientImpl.java | 37 ++++
.../agent/actions/TestActionExists.groovy | 15 +-
.../framework/AgentCommandTestBase.groovy | 32 +---
.../funtest/framework/CommandTestBase.groovy | 167 +++++++++++++++----
.../funtest/lifecycle/AMFailuresIT.groovy | 54 ++----
.../lifecycle/AgentClusterLifecycleIT.groovy | 7 +-
.../funtest/lifecycle/AgentFailures2IT.groovy | 33 +---
.../funtest/lifecycle/AgentFailuresIT.groovy | 41 +----
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 3 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 37 +---
.../src/test/resources/log4j.properties | 2 +-
12 files changed, 261 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
[04/10] git commit: SLIDER-148 one final unit test on the exists
command
Posted by st...@apache.org.
SLIDER-148 one final unit test on the exists command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b9f8db01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b9f8db01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b9f8db01
Branch: refs/heads/develop
Commit: b9f8db01c731ca62a8c83b24c4f6cd0e9a6861a2
Parents: ec07bdc
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 18:57:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:54 2014 +0100
----------------------------------------------------------------------
.../groovy/org/apache/slider/agent/actions/TestActionExists.groovy | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b9f8db01/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 2645e27..277fa7d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -124,6 +124,7 @@ class TestActionExists extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
+ assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, args)
}
}
[06/10] git commit: SLIDER-460 test AgentFailuresIT working
Posted by st...@apache.org.
SLIDER-460 test AgentFailuresIT working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/802ff46e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/802ff46e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/802ff46e
Branch: refs/heads/develop
Commit: 802ff46eeeb9881a2207c157030361cb2b57990d
Parents: 9c5ac0f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 11:17:54 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:55 2014 +0100
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 34 ++++++++++++--------
.../funtest/lifecycle/AMFailuresIT.groovy | 2 +-
.../funtest/lifecycle/AgentFailures2IT.groovy | 2 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 2 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 4 +--
5 files changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/802ff46e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 4b75c56..fb1a980 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -28,6 +28,7 @@ import org.apache.hadoop.util.ExitUtil
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.StatusKeys
import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
@@ -755,9 +756,7 @@ abstract class CommandTestBase extends SliderTestUtils {
])
assert statusFile.exists()
- cd = new ClusterDescription();
- cd.fromFile(statusFile)
- return cd
+ return ClusterDescription.fromFile(statusFile)
} finally {
statusFile.delete()
}
@@ -766,36 +765,45 @@ abstract class CommandTestBase extends SliderTestUtils {
public int queryRequestedCount(String application, String role) {
ClusterDescription cd = execStatus(application)
+ if (cd.statistics.size() == 0) {
+ log.debug("No statistics entries")
+ }
+
if (!cd.statistics[role]) {
+ log.debug("No stats for role $role")
return 0;
}
def statsForRole = cd.statistics[role]
- def requested = statsForRole["containers.requested"]
- assert null != statsForRole["containers.requested"]
- int requestedCount = requested
- return requestedCount
+ def requested = statsForRole[StatusKeys.STATISTICS_CONTAINERS_REQUESTED]
+ assert null != statsForRole[StatusKeys.STATISTICS_CONTAINERS_REQUESTED]
+ return requested
}
- boolean hasRequestedContainerCountExceeded(Map<String, String> args) {
+ boolean hasRequestedContainerCountReached(Map<String, String> args) {
String application = args['application']
String role = args['role']
int expectedCount = args['limit'].toInteger();
- return queryRequestedCount(application, role) >= expectedCount
+
+ int requestedCount = queryRequestedCount(application, role)
+ log.debug("requested count = $requestedCount; expected=$expectedCount")
+ return requestedCount >= expectedCount
}
- void expectContainerCountExceeded(String application, String role, int limit) {
+ void expectContainerRequestedCountReached(String application, String role, int limit) {
repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 50,
- 1000 * 10,
+ this.&hasRequestedContainerCountReached,
+ 90,
+ 1000,
[limit : Integer.toString(limit),
role : role,
application: application],
true,
"countainer count not reached") {
describe "container count not reached"
+ ClusterDescription cd = execStatus(application);
+ log.info("Parsed status \n$cd")
status(application).dumpOutput()
};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/802ff46e/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index f36950f..10a3dfe 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -67,7 +67,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
- expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1)
// Wait for 20 secs for AM and agent to both reach STARTED state
sleep(1000 * 20)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/802ff46e/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 9fd34f6..9b35fa4 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -58,7 +58,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
- expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 3)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
sleep(1000 * 20)
def cd = execStatus(APPLICATION_NAME)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/802ff46e/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index ab6a811..a4eb1a2 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -58,7 +58,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 2)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2)
sleep(1000 * 20)
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/802ff46e/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index f41461c..eb5baae 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -88,7 +88,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 1 )
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1 )
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
//flex
@@ -102,7 +102,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// sleep till the new instance starts
ensureApplicationIsUp(APPLICATION_NAME)
- expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 3)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
sleep(1000 * 20)
[07/10] git commit: SLIDER-460 probes working more reliably
Posted by st...@apache.org.
SLIDER-460 probes working more reliably
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9c5ac0ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9c5ac0ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9c5ac0ff
Branch: refs/heads/develop
Commit: 9c5ac0ffa0c4ea90d50ea5cbce71278b3a6d4098
Parents: 75030d2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 10:48:42 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:55 2014 +0100
----------------------------------------------------------------------
.../framework/AgentCommandTestBase.groovy | 79 +------------
.../funtest/framework/CommandTestBase.groovy | 117 +++++++++++++++++--
.../lifecycle/AgentClusterLifecycleIT.groovy | 4 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 1 -
.../src/test/resources/log4j.properties | 2 +-
5 files changed, 113 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c5ac0ff/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 4a46f8b..8af51b4 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,7 +21,6 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -125,77 +124,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
}
- public static void logShell(SliderShell shell) {
- shell.dumpOutput();
- }
-
-
- public ClusterDescription execStatus(String application) {
- ClusterDescription cd
- File statusFile = File.createTempFile("status", ".json")
- try {
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- application,
- ARG_OUTPUT, statusFile.absolutePath
- ])
-
- assert statusFile.exists()
- cd = new ClusterDescription();
- cd.fromFile(statusFile)
- return cd
- } finally {
- statusFile.delete()
- }
- }
-
- public int queryRequestedCount(String application, String role) {
- ClusterDescription cd = execStatus(application)
- int requestedCount = cd.statistics[role]["containers.requested"]
- return requestedCount
- }
-
- boolean hasRequestedContainerCountExceeded(Map<String, String> args) {
- String application = args['application']
- String role = args['role']
- int expectedCount = args['limit'].toInteger();
- return queryRequestedCount(application, role) >= expectedCount
- }
-
- void expectContainerCountExceeded(String application, String role, int limit) {
-
- repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 50,
- 1000 * 10,
- [limit : Integer.toString(limit),
- role : role,
- application: application],
- true,
- "countainer count not reached") {
- status(application).dumpOutput()
- };
-
- }
- public ClusterDescription expectContainersLive(String clustername,
- String component,
- int count) {
- ClusterDescription cd = execStatus(clustername)
- assertContainersLive(cd, component, count)
- return cd;
- }
-
- public static void assertContainersLive(ClusterDescription clusterDescription,
- String component, int count) {
- log.info("Asserting component count.")
- int instanceCount = clusterDescription.instances[component].size()
- if (count != instanceCount) {
- log.warn(clusterDescription.toString())
- }
- assert count == instanceCount
- }
-
public static String findLineEntry(SliderShell shell, String[] locaters) {
int index = 0;
def output = shell.out
@@ -279,12 +207,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
return
}
- log.info "Cleaning app instance, if exists, by name " + applicationName
+ describe "Teardown app instance " + applicationName
+ // forced freeze with wait
teardown(applicationName)
-
- // sleep till the instance is frozen
- sleep(1000 * 3)
-
SliderShell shell = slider([
ACTION_DESTROY,
applicationName])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c5ac0ff/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 44d07d8..4b75c56 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -37,7 +37,6 @@ import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.client.SliderClient
import org.apache.slider.test.SliderTestUtils
-import org.junit.Assert
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
@@ -45,6 +44,7 @@ import org.junit.rules.Timeout
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import static org.apache.slider.common.SliderExitCodes.*
+import static org.apache.slider.core.main.LauncherExitCodes.*
import static org.apache.slider.funtest.framework.FuntestProperties.*
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.common.params.SliderActions.*
@@ -169,6 +169,20 @@ abstract class CommandTestBase extends SliderTestUtils {
"and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
}
+ public static void assertContainersLive(ClusterDescription clusterDescription,
+ String component, int count) {
+ log.info("Asserting component count.")
+ int instanceCount = clusterDescription.instances[component].size()
+ if (count != instanceCount) {
+ log.warn(clusterDescription.toString())
+ }
+ assert count == instanceCount
+ }
+
+ public static void logShell(SliderShell shell) {
+ shell.dumpOutput();
+ }
+
/**
* give the test thread a name
*/
@@ -333,7 +347,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
static SliderShell freezeForce(String name) {
- freeze(name, [ARG_FORCE])
+ freeze(name, [ARG_FORCE, ARG_WAIT, "10000"])
}
static SliderShell killContainer(String name, String containerID) {
@@ -643,9 +657,12 @@ abstract class CommandTestBase extends SliderTestUtils {
sleep(5000)
ensureApplicationIsUp(cluster)
+
+/*
def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
DEFAULT_AM_RESTART_SLEEP_TIME)
sleep(sleeptime)
+*/
ClusterDescription status
status = sliderClient.clusterDescription
@@ -653,18 +670,19 @@ abstract class CommandTestBase extends SliderTestUtils {
}
protected void ensureApplicationIsUp(String application) {
- repeatUntilTrue(this.&isApplicationUp,
+ repeatUntilTrue(this.&isApplicationRunning,
SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
1000,
[application: application],
true,
'Application did not start, failing test.') {
+ describe "final state of app that tests say is not up"
exists(application,true).dumpOutput()
}
}
- protected boolean isApplicationUp(Map<String, String> args) {
+ protected boolean isApplicationRunning(Map<String, String> args) {
String applicationName = args['application'];
return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
}
@@ -686,14 +704,30 @@ abstract class CommandTestBase extends SliderTestUtils {
return shell.ret == 0
}
- protected void repeatUntilTrue(Closure closure,
+ /**
+ * Repeat a probe until it succeeds, if it does not execute a failure
+ * closure then raise an exception with the supplied message
+ * @param probe probe
+ * @param maxAttempts max number of attempts
+ * @param sleepDur sleep between failing attempts
+ * @param args map of arguments to the probe
+ * @param failIfUnsuccessful if the probe fails after all the attempts
+ * —should it raise an exception
+ * @param failureMessage message to include in exception raised
+ * @param failureHandler closure to invoke prior to the failure being raised
+ */
+ protected void repeatUntilTrue(Closure probe,
int maxAttempts, int sleepDur, Map args,
- boolean failIfUnsuccessful = false, String message,
+ boolean failIfUnsuccessful = false,
+ String failureMessage,
Closure failureHandler) {
int attemptCount = 0
+ boolean succeeded = false;
while (attemptCount < maxAttempts) {
- if (closure(args)) {
+ if (probe(args)) {
// finished
+ log.debug("Success after $attemptCount attempt(s)")
+ succeeded = true;
break
};
attemptCount++;
@@ -701,12 +735,77 @@ abstract class CommandTestBase extends SliderTestUtils {
sleep(sleepDur)
}
- if (failIfUnsuccessful & attemptCount != maxAttempts) {
+ if (failIfUnsuccessful & !succeeded) {
if (failureHandler) {
failureHandler()
}
- fail(message)
+ fail(failureMessage)
+ }
+ }
+
+ public ClusterDescription execStatus(String application) {
+ ClusterDescription cd
+ File statusFile = File.createTempFile("status", ".json")
+ try {
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ application,
+ ARG_OUTPUT, statusFile.absolutePath
+ ])
+
+ assert statusFile.exists()
+ cd = new ClusterDescription();
+ cd.fromFile(statusFile)
+ return cd
+ } finally {
+ statusFile.delete()
}
}
+ public int queryRequestedCount(String application, String role) {
+ ClusterDescription cd = execStatus(application)
+
+ if (!cd.statistics[role]) {
+ return 0;
+ }
+ def statsForRole = cd.statistics[role]
+
+ def requested = statsForRole["containers.requested"]
+ assert null != statsForRole["containers.requested"]
+ int requestedCount = requested
+ return requestedCount
+ }
+
+ boolean hasRequestedContainerCountExceeded(Map<String, String> args) {
+ String application = args['application']
+ String role = args['role']
+ int expectedCount = args['limit'].toInteger();
+ return queryRequestedCount(application, role) >= expectedCount
+ }
+
+ void expectContainerCountExceeded(String application, String role, int limit) {
+
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 50,
+ 1000 * 10,
+ [limit : Integer.toString(limit),
+ role : role,
+ application: application],
+ true,
+ "countainer count not reached") {
+ describe "container count not reached"
+ status(application).dumpOutput()
+ };
+
+ }
+
+ public ClusterDescription expectContainersLive(String clustername,
+ String component,
+ int count) {
+ ClusterDescription cd = execStatus(clustername)
+ assertContainersLive(cd, component, count)
+ return cd;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c5ac0ff/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 0a5163a..dfdbf06 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -147,7 +147,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
ARG_MESSAGE, "forced-freeze-in-test"
])
- describe " >>> Cluster is now frozen - 2nd time."
+ describe " >>> Cluster is now force frozen - 2nd time."
//cluster is no longer live
exists(0, CLUSTER, false)
@@ -165,9 +165,9 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
describe " >>> Cluster is now thawed - 2nd time."
- ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
describe " >>> Kill AM and wait for restart."
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
def restarted = status.getInfo(
StatusKeys.INFO_CONTAINERS_AM_RESTART)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c5ac0ff/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index be8614c..ab6a811 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -25,7 +25,6 @@ import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c5ac0ff/slider-funtest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/resources/log4j.properties b/slider-funtest/src/test/resources/log4j.properties
index a552a55..65135ca 100644
--- a/slider-funtest/src/test/resources/log4j.properties
+++ b/slider-funtest/src/test/resources/log4j.properties
@@ -42,7 +42,7 @@ log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
log4j.logger.org.apache.hadoop.hdfs=WARN
-
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=FATAL
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
[03/10] git commit: SLIDER-554 slide exists --state may pick up
KILLED/FINISHED app reports and then fail
Posted by st...@apache.org.
SLIDER-554 slide exists --state may pick up KILLED/FINISHED app reports and then fail
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0254e9d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0254e9d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0254e9d0
Branch: refs/heads/develop
Commit: 0254e9d00af363eba280c05824b8ae8b67c3873c
Parents: 1e4edc2
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 22 12:34:14 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:54 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 57 ++++++++++++--------
.../slider/client/SliderYarnClientImpl.java | 22 ++++++++
.../agent/actions/TestActionExists.groovy | 16 ++++--
3 files changed, 69 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0254e9d0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e89ca65..6ea305e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1752,36 +1752,49 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("Application {} exists", name);
return EXIT_SUCCESS;
}
-
+
//test for liveness/state
- ApplicationReport instance = findInstance(name);
+ boolean inDesiredState = false;
+ ApplicationReport instance;
+ instance = findInstance(name);
if (instance == null) {
log.info("Application {} not running", name);
return EXIT_FALSE;
- } else {
- // the app exists, but it may be in a terminated state
- SliderUtils.OnDemandReportStringifier report =
- new SliderUtils.OnDemandReportStringifier(instance);
- YarnApplicationState appstate =
- instance.getYarnApplicationState();
- boolean inDesiredState;
- if (checkLive) {
- inDesiredState =
+ }
+ if (checkLive) {
+ // the app exists, check that it is not in any terminated state
+ YarnApplicationState appstate = instance.getYarnApplicationState();
+ inDesiredState =
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
- } else {
- state = state.toUpperCase(Locale.ENGLISH);
- inDesiredState = state.equals(appstate.toString());
- }
- if (!inDesiredState) {
- //cluster in the list of apps but not running
- log.info("Application {} found but is in wrong state {}", name, appstate);
- log.debug("State {}", report);
- return EXIT_FALSE;
+ } else {
+ // scan for instance in single --state state
+ List<ApplicationReport> userInstances = yarnClient.listInstances("");
+ state = state.toUpperCase(Locale.ENGLISH);
+ YarnApplicationState desiredState =
+ YarnApplicationState.valueOf(state);
+ ApplicationReport foundInstance =
+ yarnClient.findClusterInInstanceList(userInstances, name, desiredState);
+ if (foundInstance != null) {
+ // found in selected state: success
+ inDesiredState = true;
+ // mark this as the instance to report
+ instance = foundInstance;
}
- log.info("Application {} is {}\n{}", name, appstate, report);
}
- return EXIT_SUCCESS;
+ SliderUtils.OnDemandReportStringifier report =
+ new SliderUtils.OnDemandReportStringifier(instance);
+ if (!inDesiredState) {
+ //cluster in the list of apps but not running
+ log.info("Application {} found but is in wrong state {}", name,
+ instance.getYarnApplicationState());
+ log.debug("State {}", report);
+ return EXIT_FALSE;
+ } else {
+ log.info("Application {} is {}\n{}", name,
+ instance.getYarnApplicationState(), report);
+ return EXIT_SUCCESS;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0254e9d0/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 3151a09..c793bd1 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -250,6 +250,12 @@ public class SliderYarnClientImpl extends YarnClientImpl {
return results;
}
+ /**
+ * Find a cluster in the instance list; biased towards live instances
+ * @param instances
+ * @param appname
+ * @return
+ */
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
ApplicationReport found = null;
@@ -268,5 +274,21 @@ public class SliderYarnClientImpl extends YarnClientImpl {
return found;
}
+ public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
+ String appname,
+ YarnApplicationState desiredState) {
+ ApplicationReport found = null;
+ ApplicationReport foundAndLive = null;
+ for (ApplicationReport app : instances) {
+ if (app.getName().equals(appname)) {
+ if (app.getYarnApplicationState().equals(desiredState)) {
+ return app;
+ }
+ }
+ }
+ // nothing found in desired state
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0254e9d0/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 277fa7d..202734c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -112,9 +112,9 @@ class TestActionExists extends AgentMiniClusterTestBase {
assert 0 == sliderClient.actionExists(clustername, true)
// assert that the cluster is in the running state
- ActionExistsArgs args = new ActionExistsArgs()
- args.state = YarnApplicationState.RUNNING.toString()
- assert 0 == sliderClient.actionExists(clustername, args)
+ ActionExistsArgs running = new ActionExistsArgs()
+ running.state = YarnApplicationState.RUNNING.toString()
+ assert 0 == sliderClient.actionExists(clustername, running)
// stop the cluster
clusterActionFreeze(sliderClient, clustername)
@@ -124,7 +124,15 @@ class TestActionExists extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, args)
+ assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, running)
+
+ // verify that on a cluster thaw the existence probes still work, that is
+ // they do not discover the previous instance and return false when there
+ // is actually a live cluster
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ addToTeardown(launcher2)
+ assert 0 == sliderClient.actionExists(clustername, running)
+ assert 0 == sliderClient.actionExists(clustername, true)
}
}
[10/10] git commit: SLIDER-561 AM Web UI doesn't show statistics for
dynamic roles
Posted by st...@apache.org.
SLIDER-561 AM Web UI doesn't show statistics for dynamic roles
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5d9a053b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5d9a053b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5d9a053b
Branch: refs/heads/develop
Commit: 5d9a053b29a53e1cdf5fc2ffc3f6c62d99097228
Parents: c87b082
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 11:40:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:40:30 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/web/WebAppApi.java | 2 +-
.../server/appmaster/web/WebAppApiImpl.java | 50 +++-----------------
2 files changed, 8 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d9a053b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index b7e749e..c8c47c9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -57,7 +57,7 @@ public interface WebAppApi {
* Generate a mapping from role name to its {@link RoleStatus}. Be aware that this
* is a computed value and not just a getter
*/
- public Map<String,RoleStatus> getRoleStatusByName();
+ public Map<String, RoleStatus> getRoleStatusByName();
/**
* Returns an interface that can support the agent-based REST operations.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d9a053b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 4c0aa6a..cacdb3a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -99,50 +99,14 @@ public class WebAppApiImpl implements WebAppApi {
* @see org.apache.slider.server.appmaster.web.WebAppApi#getRoleStatusByName()
*/
@Override
- public TreeMap<String,RoleStatus> getRoleStatusByName() {
- Map<Integer,ProviderRole> rolesById = rolesById(provider.getRoles());
- Map<Integer,RoleStatus> status = appState.getRoleStatusMap();
-
- return getRoleStatusesByName(rolesById, status);
- }
-
- /**
- * Get the ProviderRoles by their index
- * @param roles
- * @return
- */
- private Map<Integer,ProviderRole> rolesById(List<ProviderRole> roles) {
- Map<Integer,ProviderRole> rolesById = new HashMap<Integer,ProviderRole>();
- rolesById.put(SliderKeys.ROLE_AM_PRIORITY_INDEX, AM_ROLE_NAME);
-
- for (ProviderRole role : roles) {
- rolesById.put(role.id, role);
- }
-
- return rolesById;
- }
-
- /**
- * Join the ProviderRole by their ID with the RoleStatus by their ID, to get the RoleStatus by role name.
- * @param rolesById
- * @param statusById
- * @return A Map of RoleStatus by the role name
- */
- private TreeMap<String, RoleStatus> getRoleStatusesByName(Map<Integer, ProviderRole> rolesById,
- Map<Integer, RoleStatus> statusById) {
- TreeMap<String, RoleStatus> statusByName = new TreeMap<String, RoleStatus>();
- for (Entry<Integer, ProviderRole> role : rolesById.entrySet()) {
- final RoleStatus status = statusById.get(role.getKey());
-
- if (null == status) {
- log.error("Found ID ({}) which has no known ProviderRole",
- role.getKey());
- } else {
- statusByName.put(role.getValue().name, status);
- }
+ public Map<String,RoleStatus> getRoleStatusByName() {
+ List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
+ TreeMap<String, RoleStatus> map =
+ new TreeMap<String, RoleStatus>();
+ for (RoleStatus status : roleStatuses) {
+ map.put(status.getName(), status);
}
-
- return statusByName;
+ return map;
}
@Override
[02/10] git commit: SLIDER-460 factor out startup logic from test
cases; add --wait arg to launcher so the client can do most of the waiting
Posted by st...@apache.org.
SLIDER-460 factor out startup logic from test cases; add --wait arg to launcher so the client can do most of the waiting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fe5ded1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fe5ded1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fe5ded1c
Branch: refs/heads/develop
Commit: fe5ded1c271e5f07d8728211159794c324f3fd0e
Parents: 0254e9d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 22 12:35:31 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:54 2014 +0100
----------------------------------------------------------------------
.../framework/AgentCommandTestBase.groovy | 15 +++++++++
.../funtest/framework/CommandTestBase.groovy | 34 +++++++++++++-------
.../funtest/lifecycle/AMFailuresIT.groovy | 9 +-----
.../funtest/lifecycle/AgentFailures2IT.groovy | 10 +-----
.../funtest/lifecycle/AgentFailuresIT.groovy | 15 ++-------
.../AppsThroughAgentQueueAndLabelsIT.groovy | 17 ++--------
6 files changed, 44 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index effea44..4a46f8b 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -163,6 +163,21 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
return queryRequestedCount(application, role) >= expectedCount
}
+ void expectContainerCountExceeded(String application, String role, int limit) {
+
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 50,
+ 1000 * 10,
+ [limit : Integer.toString(limit),
+ role : role,
+ application: application],
+ true,
+ "countainer count not reached") {
+ status(application).dumpOutput()
+ };
+
+ }
public ClusterDescription expectContainersLive(String clustername,
String component,
int count) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index cabf927..44d07d8 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -34,10 +34,10 @@ import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.client.SliderClient
import org.apache.slider.test.SliderTestUtils
+import org.junit.Assert
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
@@ -584,14 +584,16 @@ abstract class CommandTestBase extends SliderTestUtils {
List<String> commands = [
ACTION_CREATE, name,
ARG_TEMPLATE, appTemplate,
- ARG_RESOURCES, resourceTemplate
+ ARG_RESOURCES, resourceTemplate,
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME)
+
]
-
+
maybeAddCommandOption(commands,
[ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME],
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME));
maybeAddCommandOption(commands,
- [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR ],
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR],
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR));
maybeAddCommandOption(commands,
[ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH],
@@ -657,7 +659,9 @@ abstract class CommandTestBase extends SliderTestUtils {
1000,
[application: application],
true,
- 'Application did not start, failing test.')
+ 'Application did not start, failing test.') {
+ exists(application,true).dumpOutput()
+ }
}
protected boolean isApplicationUp(Map<String, String> args) {
@@ -682,21 +686,27 @@ abstract class CommandTestBase extends SliderTestUtils {
return shell.ret == 0
}
- protected void repeatUntilTrue(Closure c, int maxAttempts, int sleepDur, Map args,
- boolean failIfUnsuccessful = false, String message = "") {
+ protected void repeatUntilTrue(Closure closure,
+ int maxAttempts, int sleepDur, Map args,
+ boolean failIfUnsuccessful = false, String message,
+ Closure failureHandler) {
int attemptCount = 0
while (attemptCount < maxAttempts) {
- if (c(args)) {
+ if (closure(args)) {
+ // finished
break
};
attemptCount++;
- if (failIfUnsuccessful) {
- assert attemptCount != maxAttempts, message
- }
-
sleep(sleepDur)
}
+
+ if (failIfUnsuccessful & attemptCount != maxAttempts) {
+ if (failureHandler) {
+ failureHandler()
+ }
+ fail(message)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 323bf7d..f36950f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -67,14 +67,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
-
- repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 20,
- 1000 * 10,
- [limit : '1',
- role : COMMAND_LOGGER,
- application: APPLICATION_NAME]);
+ expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 1)
// Wait for 20 secs for AM and agent to both reach STARTED state
sleep(1000 * 20)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index f1d5f84..9fd34f6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -58,15 +58,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
-
- repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 20,
- 1000 * 10,
- [limit: '3',
- role: COMMAND_LOGGER ,
- application:APPLICATION_NAME]);
-
+ expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 3)
sleep(1000 * 20)
def cd = execStatus(APPLICATION_NAME)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 97daff1..be8614c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -47,8 +47,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testAgentFailRegistrationOnce() throws Throwable {
if (!AGENTTESTS_ENABLED) {
- log.info "TESTS are not run."
- return
+ skip("Agent tests are not run.")
}
cleanup(APPLICATION_NAME)
@@ -60,17 +59,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasRequestedContainerCountExceeded,
- 15,
- 1000 * 10,
- [
- application:APPLICATION_NAME,
- role:COMMAND_LOGGER,
- limit: '2'
- ]);
-
+ expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 2)
sleep(1000 * 20)
-
+ assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe5ded1c/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index 1a5c5ff..f41461c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -88,14 +88,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 50,
- 1000 * 10,
- [limit : '1',
- role : COMMAND_LOGGER,
- application: APPLICATION_NAME]);
-
+ expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 1 )
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
//flex
@@ -109,14 +102,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// sleep till the new instance starts
ensureApplicationIsUp(APPLICATION_NAME)
+ expectContainerCountExceeded(APPLICATION_NAME, COMMAND_LOGGER, 3)
- repeatUntilTrue(
- this.&hasRequestedContainerCountExceeded,
- 20,
- 1000 * 10,
- [limit : '3',
- role : COMMAND_LOGGER,
- application: APPLICATION_NAME]);
sleep(1000 * 20)
def cd = execStatus(APPLICATION_NAME)
[05/10] git commit: SLIDER-460 move all tests from scanning stdout to
parsing the JSON from cluster status operations and examining it via the
ClusterDescription class
Posted by st...@apache.org.
SLIDER-460 move all tests from scanning stdout to parsing the JSON from cluster status operations and examining it via the ClusterDescription class
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1e4edc29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1e4edc29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1e4edc29
Branch: refs/heads/develop
Commit: 1e4edc299800a471541f5ae49482547b1cda7528
Parents: b9f8db0
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 18:58:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 23 11:35:54 2014 +0100
----------------------------------------------------------------------
.../framework/AgentCommandTestBase.groovy | 66 +++++++++++++++-----
.../funtest/framework/CommandTestBase.groovy | 24 +++----
.../funtest/lifecycle/AMFailuresIT.groovy | 61 +++++++-----------
.../funtest/lifecycle/AgentFailures2IT.groovy | 37 +++--------
.../funtest/lifecycle/AgentFailuresIT.groovy | 41 +++---------
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 3 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 48 ++++++--------
7 files changed, 118 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 69523aa..effea44 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -128,25 +129,56 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
shell.dumpOutput();
}
- public static void assertComponentCount(String component, int count, SliderShell shell) {
- log.info("Asserting component count.")
- int instanceCount = getComponentCount(component, shell)
- assert count == instanceCount, 'Instance count for component did not match expected.'
- }
-
- public static int getComponentCount(String component, SliderShell shell) {
- String entry = findLineEntry(shell, ["instances", component] as String[])
- int instanceCount = 0
- if (!SliderUtils.isUnset(entry)) {
- log.info(entry)
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
- }
+
+ public ClusterDescription execStatus(String application) {
+ ClusterDescription cd
+ File statusFile = File.createTempFile("status", ".json")
+ try {
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ application,
+ ARG_OUTPUT, statusFile.absolutePath
+ ])
+
+ assert statusFile.exists()
+ cd = new ClusterDescription();
+ cd.fromFile(statusFile)
+ return cd
+ } finally {
+ statusFile.delete()
}
+ }
- return instanceCount
+ public int queryRequestedCount(String application, String role) {
+ ClusterDescription cd = execStatus(application)
+ int requestedCount = cd.statistics[role]["containers.requested"]
+ return requestedCount
+ }
+
+ boolean hasRequestedContainerCountExceeded(Map<String, String> args) {
+ String application = args['application']
+ String role = args['role']
+ int expectedCount = args['limit'].toInteger();
+ return queryRequestedCount(application, role) >= expectedCount
+ }
+
+ public ClusterDescription expectContainersLive(String clustername,
+ String component,
+ int count) {
+ ClusterDescription cd = execStatus(clustername)
+ assertContainersLive(cd, component, count)
+ return cd;
+ }
+
+ public static void assertContainersLive(ClusterDescription clusterDescription,
+ String component, int count) {
+ log.info("Asserting component count.")
+ int instanceCount = clusterDescription.instances[component].size()
+ if (count != instanceCount) {
+ log.warn(clusterDescription.toString())
+ }
+ assert count == instanceCount
}
public static String findLineEntry(SliderShell shell, String[] locaters) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 9230a7a..cabf927 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -639,39 +639,29 @@ abstract class CommandTestBase extends SliderTestUtils {
])
+ sleep(5000)
+ ensureApplicationIsUp(cluster)
def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
DEFAULT_AM_RESTART_SLEEP_TIME)
sleep(sleeptime)
ClusterDescription status
- try {
- // am should have restarted it by now
- // cluster is live
- exists(0, cluster, true)
-
- status = sliderClient.clusterDescription
- } catch (SliderException e) {
- if (e.exitCode == EXIT_BAD_STATE) {
- log.error(
- "Property $YarnConfiguration.RM_AM_MAX_ATTEMPTS may be too low")
- }
- throw e;
- }
+ status = sliderClient.clusterDescription
return status
}
- protected void ensureApplicationIsUp(String clusterName) {
+ protected void ensureApplicationIsUp(String application) {
repeatUntilTrue(this.&isApplicationUp,
SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
1000,
- ['arg1': clusterName],
+ [application: application],
true,
- 'Application did not start, aborting test.')
+ 'Application did not start, failing test.')
}
protected boolean isApplicationUp(Map<String, String> args) {
- String applicationName = args['arg1'];
+ String applicationName = args['application'];
return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 159e2d6..323bf7d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -67,18 +67,27 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit : '1',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
+
// Wait for 20 secs for AM and agent to both reach STARTED state
sleep(1000 * 20)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
- assert live != null && live.isInteger() && live.toInteger() == 1,
- 'At least 1 container must be live now'
+ def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ def loggerInstances = cd.instances[COMMAND_LOGGER]
+ assert loggerInstances.size() == 1
+
+ def loggerStats = cd.statistics[COMMAND_LOGGER]
+
+ def origRequested = loggerStats["containers.requested"]
+ assert origRequested >= 2
+ assert loggerStats["containers.live"] == 1
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
assertSuccess(shell)
@@ -98,42 +107,16 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// Wait until AM comes back up and verify container count again
ensureApplicationIsUp(APPLICATION_NAME)
- // There should be exactly 1 live container
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
- assert live != null && live.isInteger() && live.toInteger() == 1,
- 'At least 1 container must be live now'
- log.info("After AM KILL: agent container is still live")
+ // There should be exactly 1 live logger container
+ def cd2 = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
// No new containers should be requested for the agents
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() == 0,
+ def loggerStats2 = cd2.statistics[COMMAND_LOGGER]
+ assert origRequested == loggerStats2["containers.requested"],
'No new agent containers should be requested'
- log.info("After AM KILL: no new agent containers were requested")
-
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
- assertSuccess(shell)
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
protected void killAMUsingAmSuicide() {
SliderShell shell = slider(EXIT_SUCCESS,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index a5cf5be..f1d5f84 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -59,41 +59,22 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 20, 1000 * 10, ['arg1': '3']);
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit: '3',
+ role: COMMAND_LOGGER ,
+ application:APPLICATION_NAME]);
sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell,
- ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
- 'At least 2 containers must be requested'
+ def cd = execStatus(APPLICATION_NAME)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
- assertSuccess(shell)
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a384170..97daff1 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -39,7 +39,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String APP_TEMPLATE2 =
"../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
-
@After
public void destroyCluster() {
cleanup(APPLICATION_NAME)
@@ -61,40 +60,20 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
-
- sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
+ repeatUntilTrue(this.&hasRequestedContainerCountExceeded,
+ 15,
+ 1000 * 10,
[
- ACTION_STATUS,
- APPLICATION_NAME])
+ application:APPLICATION_NAME,
+ role:COMMAND_LOGGER,
+ limit: '2'
+ ]);
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 2,
- 'At least 2 containers must be requested'
+ sleep(1000 * 20)
+ def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
-
- assertSuccess(shell)
}
-
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 9389f80..80b5db7 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -72,9 +72,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ACTION_STATUS,
APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
- assertSuccess(shell)
// get log folders
shell = slider(EXIT_SUCCESS,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e4edc29/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index c87d507..1a5c5ff 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -88,13 +88,15 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ensureApplicationIsUp(APPLICATION_NAME)
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 50,
+ 1000 * 10,
+ [limit : '1',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 1, shell)
+ expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
//flex
slider(EXIT_SUCCESS,
@@ -106,32 +108,22 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
"2"])
// sleep till the new instance starts
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
+ ensureApplicationIsUp(APPLICATION_NAME)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ repeatUntilTrue(
+ this.&hasRequestedContainerCountExceeded,
+ 20,
+ 1000 * 10,
+ [limit : '3',
+ role : COMMAND_LOGGER,
+ application: APPLICATION_NAME]);
+
+ sleep(1000 * 20)
+ def cd = execStatus(APPLICATION_NAME)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
- assertSuccess(shell)
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
}
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- int instanceCount = getComponentCount(COMMAND_LOGGER, shell)
- log.info("Noticed instance count - " + instanceCount + " for " + COMMAND_LOGGER)
- if (instanceCount >= expectedCount) {
- return true
- }
- return false
- }
}