You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/06 15:45:29 UTC
[1/4] git commit: SLIDER-616 re-enable AMFailuresIT
Repository: incubator-slider
Updated Branches:
refs/heads/develop 81ee39b48 -> b39a73383
SLIDER-616 re-enable AMFailuresIT
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d64b2c79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d64b2c79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d64b2c79
Branch: refs/heads/develop
Commit: d64b2c79e894d28b34367d53425f281871e17e26
Parents: 61e9d1d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 5 20:28:12 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 6 11:42:47 2014 +0000
----------------------------------------------------------------------
.../apache/slider/test/SliderTestUtils.groovy | 23 ++-
.../funtest/framework/CommandTestBase.groovy | 98 +++++++---
.../funtest/lifecycle/AMFailuresIT.groovy | 29 +--
.../lifecycle/AgentClusterLifecycleIT.groovy | 6 +-
.../funtest/lifecycle/AgentFailures2IT.groovy | 2 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 2 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 2 +-
.../funtest/lifecycle/AgentRegistryIT.groovy | 8 +-
.../lifecycle/AppsThroughAgentDemo.groovy | 139 +-------------
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 181 ++++++++++++-------
.../AppsThroughAgentQueueAndLabelsIT.groovy | 2 +-
11 files changed, 238 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 1afbafc..3b1d022 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -938,12 +938,19 @@ class SliderTestUtils extends Assert {
* @param failureMessage message to include in exception raised
* @param failureHandler closure to invoke prior to the failure being raised
*/
- protected void repeatUntilSuccess(Closure probe,
- int timeout, int sleepDur,
+ protected void repeatUntilSuccess(
+ String action,
+ Closure probe,
+ int timeout,
+ int sleepDur,
Map args,
boolean failIfUnsuccessful,
String failureMessage,
Closure failureHandler) {
+ log.debug("Probe $action timelimit $timeout")
+ if (timeout < 1000) {
+ fail("Timeout $timeout too low: milliseconds are expected, not seconds")
+ }
int attemptCount = 0
boolean succeeded = false;
boolean completed = false;
@@ -961,19 +968,25 @@ class SliderTestUtils extends Assert {
attemptCount++;
completed = duration.limitExceeded
if (!completed) {
+ log.debug("Attempt $attemptCount failed")
sleep(sleepDur)
}
} else if (outcome.equals(Outcome.Fail)) {
// fast fail
+ log.debug("Fast fail of probe")
completed = true;
}
}
-
- if (failIfUnsuccessful & !succeeded) {
+ if (!succeeded) {
+ if (duration.limitExceeded) {
+ log.info("probe timed out after $timeout and $attemptCount attempts")
+ }
if (failureHandler) {
failureHandler()
}
- fail(failureMessage)
+ if (failIfUnsuccessful) {
+ fail(failureMessage)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7d369d5..1d436da 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -511,7 +511,7 @@ abstract class CommandTestBase extends SliderTestUtils {
if (!shell.outputContains(lookThisUp)) {
log.error("Missing $lookThisUp from:")
shell.dumpOutput()
- assert shell.outputContains(lookThisUp)
+ fail("Missing $lookThisUp from:\n$shell.out\n$shell.err" )
}
}
@@ -629,7 +629,7 @@ abstract class CommandTestBase extends SliderTestUtils {
File launchReportFile = null) {
if (!launchReportFile) {
- launchReportFile = createAppReportFile()
+ launchReportFile = createTempJsonFile()
}
// delete any previous copy of the file
launchReportFile.delete();
@@ -678,10 +678,19 @@ abstract class CommandTestBase extends SliderTestUtils {
return shell
}
- public static File createAppReportFile() {
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() {
+ return tmpFile(".json")
+ }
+
+ public static File tmpFile(String suffix) {
File reportFile = File.createTempFile(
"launch",
- ".json",
+ suffix,
new File("target"))
reportFile.delete()
return reportFile
@@ -737,7 +746,7 @@ abstract class CommandTestBase extends SliderTestUtils {
* @return an application report or null
*/
public static SerializedApplicationReport lookupApplication(String id) {
- File reportFile = createAppReportFile();
+ File reportFile = createTempJsonFile();
try {
def shell = lookup(id, reportFile)
if (shell.ret == 0) {
@@ -789,11 +798,15 @@ abstract class CommandTestBase extends SliderTestUtils {
slider(0, [
ACTION_AM_SUICIDE, application,
ARG_EXITCODE, "1",
- ARG_WAIT, "1000",
+ ARG_WAIT, "500",
ARG_MESSAGE, "suicide"
])
- sleep(5000)
+ // app gets accepted
+ log.info "awaiting app to enter ACCEPTED state"
+ awaitYarnApplicationAccepted(appId)
+ // app goes live
+ log.info "awaiting app to enter RUNNING state"
ensureYarnApplicationIsUp(appId)
}
@@ -802,7 +815,8 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param application application
*/
protected void ensureRegistryCallSucceeds(String application) {
- repeatUntilSuccess(this.&isRegistryAccessible,
+ repeatUntilSuccess("registry",
+ this.&isRegistryAccessible,
REGISTRY_STARTUP_TIMEOUT,
PROBE_SLEEP_TIME,
[application: application],
@@ -823,9 +837,9 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param application
*/
protected void ensureApplicationIsUp(String application) {
- repeatUntilSuccess(this.&isApplicationRunning,
- SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
- DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS) * 1000,
+ repeatUntilSuccess("await application up",
+ this.&isApplicationRunning,
+ instanceLaunchTime,
PROBE_SLEEP_TIME,
[application: application],
true,
@@ -898,9 +912,22 @@ abstract class CommandTestBase extends SliderTestUtils {
* @return
*/
- protected Outcome isYarnApplicationRunning(Map<String, String> args) {
+ protected static Outcome isYarnApplicationRunning(Map<String, String> args) {
+ String applicationId = args['applicationId'];
+ return isYarnApplicationInState(applicationId,
+ YarnApplicationState.RUNNING, true)
+ }
+
+ /**
+ * Probe callback for is the the app running or not
+ * @param args map where 'applicationId' must m
+ * @return
+ */
+ protected static Outcome isYarnApplicationInExactState(Map<String, String> args) {
String applicationId = args['applicationId'];
- return isYarnApplicationRunning(applicationId)
+ String state = args['state']
+ def desired = YarnApplicationState.valueOf(state)
+ return isYarnApplicationInState(applicationId, desired, false)
}
/**
@@ -912,8 +939,8 @@ abstract class CommandTestBase extends SliderTestUtils {
*/
public static Outcome isYarnApplicationRunning(
String applicationId) {
- YarnApplicationState yarnState = YarnApplicationState.RUNNING
- return isYarnApplicationInState(applicationId, yarnState)
+ return isYarnApplicationInState(applicationId,
+ YarnApplicationState.RUNNING, true)
}
/**
@@ -924,14 +951,14 @@ abstract class CommandTestBase extends SliderTestUtils {
* above it
*/
public static Outcome isYarnApplicationInState(
- String applicationId,
- YarnApplicationState yarnState) {
+ String applicationId, YarnApplicationState yarnState, boolean failfast) {
YarnApplicationState appState = lookupYarnAppState(applicationId)
if (yarnState == appState) {
return Outcome.Success;
}
- if (appState.ordinal() > yarnState.ordinal()) {
+ if (failfast && appState.ordinal() > yarnState.ordinal()) {
+ log.debug("App state $appState past desired state $yarnState: failing")
// app has passed beyond hope
return Outcome.Fail
}
@@ -977,13 +1004,14 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param applicationId
*/
protected void ensureYarnApplicationIsUp(String applicationId) {
- repeatUntilSuccess(this.&isYarnApplicationRunning,
+ repeatUntilSuccess("await yarn application Running",
+ this.&isYarnApplicationRunning,
instanceLaunchTime,
PROBE_SLEEP_TIME,
[applicationId: applicationId],
true,
E_LAUNCH_FAIL) {
- describe "final state of app that tests say is not up"
+ describe "final state of application"
def sar = lookupApplication(applicationId)
def message = E_LAUNCH_FAIL + "\n$sar"
@@ -993,6 +1021,27 @@ abstract class CommandTestBase extends SliderTestUtils {
}
/**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param applicationId
+ */
+ protected void awaitYarnApplicationAccepted(String applicationId) {
+ repeatUntilSuccess("Await Yarn App Accepted",
+ this.&isYarnApplicationInExactState,
+ instanceLaunchTime,
+ 1000,
+ [applicationId: applicationId,
+ state: YarnApplicationState.ACCEPTED.toString()],
+ true,
+ "application never reached accepted state") {
+ describe "app did not enter accepted"
+ def sar = lookupApplication(applicationId)
+
+ def message = 'Application did not enter ACCEPTED state' + "\n$sar"
+ log.error(message)
+ fail(message)
+ }
+ }
+ /**
* Get the expected launch time. Default is the configuration option
* {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME} and
* default value {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME}
@@ -1000,7 +1049,7 @@ abstract class CommandTestBase extends SliderTestUtils {
*/
public int getInstanceLaunchTime() {
return SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
- DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS)
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS) * 1000;
}
public String getInfoAmWebUrl(String applicationName) {
@@ -1045,6 +1094,11 @@ abstract class CommandTestBase extends SliderTestUtils {
return requested
}
+ /**
+ * Probe: has the requested container count of a specific role been reached?
+ * @param args map with: "application", "role", "limit"
+ * @return success on a match, retry if not
+ */
Outcome hasRequestedContainerCountReached(Map<String, String> args) {
String application = args['application']
String role = args['role']
@@ -1059,6 +1113,7 @@ abstract class CommandTestBase extends SliderTestUtils {
int container_launch_timeout) {
repeatUntilSuccess(
+ "await container count",
this.&hasRequestedContainerCountReached,
container_launch_timeout,
PROBE_SLEEP_TIME,
@@ -1119,6 +1174,7 @@ abstract class CommandTestBase extends SliderTestUtils {
int container_launch_timeout) {
repeatUntilSuccess(
+ "await live container count",
this.&hasLiveContainerCountReached,
container_launch_timeout,
PROBE_SLEEP_TIME,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 6238164..0544b22 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -43,7 +43,7 @@ public class AMFailuresIT extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
- private static String APPLICATION_NAME = "am-started-agents-started"
+ private static String APPLICATION_NAME = "am-failures-it"
public static final String TEST_REMOTE_SSH_KEY = "test.remote.ssh.key"
public static final String VAGRANT_CWD = "vagrant.current.working.dir"
File sshkey
@@ -61,7 +61,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable {
cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(
APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE,
@@ -84,37 +84,20 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// Now kill the AM
log.info("Killing AM now ...")
-// killAMUsingJsch()
killAmAndWaitForRestart(APPLICATION_NAME, appId)
// There should be exactly 1 live logger container
def cd2 = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
// No new containers should be requested for the agents
- def loggerStats2 = cd2.statistics[COMMAND_LOGGER]
- assert loggerStats["containers.requested"] == loggerStats2["containers.requested"],
+ def restartedStats = cd2.statistics[COMMAND_LOGGER]
+ assert restartedStats["containers.live"] == 1
+
+ assert 0==restartedStats["containers.requested"],
'No new agent containers should be requested'
assert lookupYarnAppState(appId) == YarnApplicationState.RUNNING
}
- /**
- * Allow for 2x as long as other test instances, as for AM restart we
- * need to allow for a longer delay
- */
- @Override
- int getInstanceLaunchTime() {
- return 2* super.instanceLaunchTime
- }
-
- protected void killAMUsingAmSuicide() {
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_AM_SUICIDE,
- ARG_MESSAGE, "testAMRestart",
- APPLICATION_NAME])
- logShell(shell)
- }
-
protected void killAMUsingVagrantShell() {
String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
assert hostname
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index a0cb6ca..0bc4e39 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -67,7 +67,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
@@ -158,7 +158,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
list(-1, [ARG_STATE, "running"])
list( 0, [ARG_STATE, "FINISHED"])
- def thawReport = createAppReportFile()
+ def thawReport = createTempJsonFile()
//start then stop the cluster
thaw(CLUSTER,
[
@@ -198,7 +198,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
//start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
- def thawReport2 = createAppReportFile()
+ def thawReport2 = createTempJsonFile()
//start then stop the cluster
thaw(CLUSTER,
[
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index bfae6ec..39b5d6c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -50,7 +50,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(
APPLICATION_NAME, APP_TEMPLATE3, APP_RESOURCE,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a6930b2..7d1be89 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -52,7 +52,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(
APPLICATION_NAME,
APP_TEMPLATE2,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index 791123c..2e1ca0f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -60,7 +60,7 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
assert 0 != exists(CLUSTER).ret
// create an AM which fails to launch
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index fe63660..4f9701c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -24,13 +24,10 @@ import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.Endpoint
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.core.exceptions.NotFoundException
import org.apache.slider.test.Outcome
import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
@@ -66,7 +63,7 @@ public class AgentRegistryIT extends AgentCommandTestBase
public void testAgentRegistry() throws Throwable {
describe("Create a 0-role cluster and make registry queries against it")
def clusterpath = buildClusterPath(CLUSTER)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
@@ -143,7 +140,8 @@ public class AgentRegistryIT extends AgentCommandTestBase
exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
- repeatUntilSuccess(this.&probeForEntryMissing, 10000, 1000,
+ repeatUntilSuccess("probe for missing registry entry",
+ this.&probeForEntryMissing, 10000, 1000,
[path: appPath],
true,
"registry entry never deleted") {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
index 1d65ce7..0f940cf 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
@@ -29,138 +29,17 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
+/**
+ * For a quick demo of a slider app; this starts the apps through agent test but
+ * neglects to tear it down afterwards
+ */
@CompileStatic
@Slf4j
-public class AppsThroughAgentDemo extends AgentCommandTestBase
-implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
-
- private static String COMMAND_LOGGER = "COMMAND_LOGGER"
- private static String APPLICATION_NAME = "agent-demo"
-
-
- @Before
- public void prepareCluster() {
- setupCluster(APPLICATION_NAME)
- }
-
- @Test
- public void testCreateFlex() throws Throwable {
- assumeAgentTestsEnabled()
-
- cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
-
- SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
- APP_TEMPLATE,
- APP_RESOURCE,
- [],
- launchReportFile)
-
- logShell(shell)
-
- def appId = ensureYarnApplicationIsUp(launchReportFile)
-
- //flex
- slider(EXIT_SUCCESS,
- [
- ACTION_FLEX,
- APPLICATION_NAME,
- ARG_COMPONENT,
- COMMAND_LOGGER,
- "2"])
-
- // sleep till the new instance starts
- sleep(1000 * 10)
-
- slider(EXIT_SUCCESS,
- [ACTION_STATUS,
- APPLICATION_NAME])
-
- assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
-
- String amWebUrl = getInfoAmWebUrl(APPLICATION_NAME)
- log.info("Dumping data from AM Web URL");
- log.info(amWebUrl.toURL().text);
-
- ensureRegistryCallSucceeds(APPLICATION_NAME)
-
- // get log folders
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_LISTEXP
- ])
- if(!containsString(shell, "container_log_dirs") ||
- !containsString(shell, "container_work_dirs")) {
- logShell(shell)
- assert fail("Should list default exports container_log_dirs or container_work_dirs")
- }
-
- // get log folders
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETEXP,
- "container_log_dirs"
- ])
- if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
- || !containsString(shell, "\"level\" : \"component\"", 2)) {
- logShell(shell)
- assert fail("Should list 2 entries for log folders")
- }
-
- // get log folders
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETEXP,
- "container_work_dirs"
- ])
- if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
- || !containsString(shell, "\"level\" : \"component\"", 2)) {
- logShell(shell)
- assert fail("Should list 2 entries for work folder")
- }
-
- // get cl-site config
- shell = slider(
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETCONF,
- "cl-site",
- ARG_FORMAT,
- "json"
- ])
-
- for (int i = 0; i < 10; i++) {
- if (shell.getRet() != EXIT_SUCCESS) {
- println "Waiting for the cl-site to show up"
- sleep(1000 * 10)
- shell = slider(
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETCONF,
- "cl-site",
- ARG_FORMAT,
- "json"])
- }
- }
- assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved"
- if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) {
- logShell(shell)
- assert fail("Should have exported cl-site")
- }
+public class AppsThroughAgentDemo extends AppsThroughAgentIT {
- assertAppRunning(appId)
+ @Override
+ void destroyCluster() {
+ super.destroyCluster()
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 5cd6dc5..8ddc38f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -26,6 +26,7 @@ import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.test.Outcome
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -43,6 +44,10 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
setupCluster(APPLICATION_NAME)
}
+ public String getApplicationName() {
+ return APPLICATION_NAME
+ }
+
@After
public void destroyCluster() {
cleanup(APPLICATION_NAME)
@@ -52,9 +57,10 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public void testCreateFlex() throws Throwable {
assumeAgentTestsEnabled()
- cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
- SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ def application = APPLICATION_NAME
+ cleanup(application)
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(application,
APP_TEMPLATE,
APP_RESOURCE,
[],
@@ -67,7 +73,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
slider(EXIT_SUCCESS,
[
ACTION_FLEX,
- APPLICATION_NAME,
+ application,
ARG_COMPONENT,
COMMAND_LOGGER,
"2"])
@@ -75,91 +81,140 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// sleep till the new instance starts
sleep(1000 * 10)
- status(0, APPLICATION_NAME)
- expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+ status(0, application)
+ expectLiveContainerCountReached(application, COMMAND_LOGGER, 2,
CONTAINER_LAUNCH_TIMEOUT)
- String amWebUrl = getInfoAmWebUrl(APPLICATION_NAME)
+ String amWebUrl = getInfoAmWebUrl(application)
log.info("Dumping data from AM Web URL");
log.info(amWebUrl.toURL().text);
- ensureRegistryCallSucceeds(APPLICATION_NAME)
+ ensureRegistryCallSucceeds(application)
+ def outfile = tmpFile(".txt")
+
+ assertAppRunning(appId)
+ def commands = [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ application,
+ ARG_LISTEXP,
+ ARG_OUTPUT,
+ outfile.absolutePath
+ ]
+
+ awaitRegistryOutfileContains(outfile, commands, "container_log_dirs")
+ awaitRegistryOutfileContains(outfile, commands, "container_work_dirs")
// get log folders
shell = slider(EXIT_SUCCESS,
[
ACTION_REGISTRY,
ARG_NAME,
- APPLICATION_NAME,
+ application,
ARG_LISTEXP])
- if(!containsString(shell, "container_log_dirs") ||
- !containsString(shell, "container_work_dirs")) {
- logShell(shell)
- assert fail("Should list default exports container_log_dirs or container_work_dirs")
- }
+ assertOutputContains(shell, "container_log_dirs")
+ assertOutputContains(shell, "container_work_dirs")
// get log folders
shell = slider(EXIT_SUCCESS,
[
ACTION_REGISTRY,
ARG_NAME,
- APPLICATION_NAME,
+ application,
ARG_GETEXP,
"container_log_dirs"])
- if (!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
- || !containsString(shell, "\"level\" : \"component\"", 2)) {
- logShell(shell)
- assert fail("Should list 2 entries for log folders")
- }
- // get log folders
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETEXP,
- "container_work_dirs"])
- if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
- || !containsString(shell, "\"level\" : \"component\"", 2)) {
- logShell(shell)
- assert fail("Should list 2 entries for work folder")
- }
+ assertOutputContains(shell, '"tag" : "COMMAND_LOGGER"', 2)
+ assertOutputContains(shell, '"level" : "level"', 2)
// get cl-site config
- shell = slider(
+
+ def getconf = [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ application,
+ ARG_GETCONF,
+ "cl-site",
+ ARG_FORMAT,
+ "json"]
+
+ def pattern = '"pattern.for.test.to.verify" : "verify this pattern"'
+
+ repeatUntilSuccess("registry",
+ this.&commandSucceeds,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
[
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETCONF,
- "cl-site",
- ARG_FORMAT,
- "json"])
-
- for (int i = 0; i < 10; i++) {
- if (shell.getRet() != EXIT_SUCCESS) {
- println "Waiting for the cl-site to show up"
- sleep(1000 * 10)
- shell = slider(0,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- APPLICATION_NAME,
- ARG_GETCONF,
- "cl-site",
- ARG_FORMAT,
- "json"])
- }
- }
- if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) {
- logShell(shell)
-
- fail("Should have exported cl-site; got " +
- "stdout" +shell.stdErrHistory +
- " \nstderr:" + shell.stdErrHistory)
+ text: "pattern",
+ command: getconf
+ ],
+ true,
+ "failed to find $pattern in output") {
+ slider(0, getconf)
+ assertOutputContains(shell, pattern)
}
assertAppRunning(appId)
}
+
+ public awaitRegistryOutfileContains(
+ File outfile,
+ List<String> commands,
+ String match) {
+ repeatUntilSuccess("registry",
+ this.&generatedFileContains,
+ REGISTRY_STARTUP_TIMEOUT * 2,
+ PROBE_SLEEP_TIME,
+ [
+ text : match,
+ filename: outfile.absolutePath,
+ command : commands
+ ],
+ true,
+ "failed to find $match in output") {
+ slider(0, commands).dumpOutput()
+ fail("no $match in \n$outfile.text")
+ }
+ }
+
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandOutputContains(Map args) {
+ String text = args['text'];
+ List<String> command = (List < String >)args['command']
+ SliderShell shell = slider(0, command)
+ return Outcome.fromBool(shell.outputContains(text))
+ }
+
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandSucceeds(Map args) {
+ List<String> command = (List<String>) args['command']
+ SliderShell shell = slider(command)
+ return Outcome.fromBool(shell.ret == 0)
+ }
+
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map
+ * @return probe outcome
+ */
+ protected Outcome generatedFileContains(Map args) {
+ List<String> command = (List<String>) args['command']
+ String text = args['text'];
+ String filename = args['filename'];
+ File f = new File(filename)
+ f.delete()
+ SliderShell shell = slider(0, command)
+ shell.dumpOutput()
+ assert f.exists()
+ return Outcome.fromBool(f.text.contains(text))
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d64b2c79/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index 51bb440..4ef3905 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -81,7 +81,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeLabelsRedAndBlueAdded()
cleanup(APPLICATION_NAME)
- File launchReportFile = createAppReportFile();
+ File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(
APPLICATION_NAME,
APP_TEMPLATE,
[4/4] git commit: Merge branch
'feature/SLIDER-616_re-enable_AMFailuresIT' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-616_re-enable_AMFailuresIT' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b39a7338
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b39a7338
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b39a7338
Branch: refs/heads/develop
Commit: b39a733836730fdc5bd7ec4e7200d18aa1dee5ea
Parents: 81ee39b 2b8dd55
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 6 14:40:00 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 6 14:40:00 2014 +0000
----------------------------------------------------------------------
.../apache/slider/test/SliderTestUtils.groovy | 23 +-
slider-funtest/pom.xml | 4 +-
.../framework/AgentCommandTestBase.groovy | 3 +-
.../funtest/framework/CommandTestBase.groovy | 248 +++++++++++++++++--
.../funtest/lifecycle/AMFailuresIT.groovy | 48 +---
.../lifecycle/AgentClusterLifecycleIT.groovy | 6 +-
.../funtest/lifecycle/AgentFailures2IT.groovy | 2 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 2 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 2 +-
.../funtest/lifecycle/AgentRegistryIT.groovy | 8 +-
.../lifecycle/AppsThroughAgentDemo.groovy | 139 +----------
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 122 ++++-----
.../AppsThroughAgentQueueAndLabelsIT.groovy | 2 +-
13 files changed, 325 insertions(+), 284 deletions(-)
----------------------------------------------------------------------
[2/4] git commit: SLIDER-616 AMFailuresIT
Posted by st...@apache.org.
SLIDER-616 AMFailuresIT
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/61e9d1d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/61e9d1d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/61e9d1d4
Branch: refs/heads/develop
Commit: 61e9d1d445bc926d2470fdf5cb8030cbf03494e2
Parents: 81ee39b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 5 15:37:55 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 6 11:42:47 2014 +0000
----------------------------------------------------------------------
slider-funtest/pom.xml | 4 +-
.../framework/AgentCommandTestBase.groovy | 3 +-
.../funtest/framework/CommandTestBase.groovy | 90 +++++++++++++++++---
.../funtest/lifecycle/AMFailuresIT.groovy | 37 +++-----
4 files changed, 96 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61e9d1d4/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index a2abd25..fbeb11d 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -122,7 +122,7 @@
<test.app.pkg.name>CMD_LOGGER</test.app.pkg.name>
<test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
<test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
- <vagrant.current.working.dir>/Users/gsaha/work/workspace/ambari-vagrant/centos6.4</vagrant.current.working.dir>
+ <vagrant.current.working.dir></vagrant.current.working.dir>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -172,8 +172,6 @@
<reporting>
<plugins>
-
-
</plugins>
</reporting>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61e9d1d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 6e05056..668a264 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -67,7 +67,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
AGENTTESTS_LABELS_RED_BLUE_DEFINED =
SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED, false)
AGENTTESTS_AM_FAILURES_ENABLED =
- SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_AM_FAILURES_ENABLED, false)
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_AM_FAILURES_ENABLED,
+ AGENTTESTS_ENABLED)
}
protected String getAppResource() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61e9d1d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 36515c5..7d369d5 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -775,12 +775,19 @@ abstract class CommandTestBase extends SliderTestUtils {
ensureApplicationIsUp(cluster)
return sliderClient.clusterDescription
}
- public ClusterDescription killAmAndWaitForRestart(
- SliderClient sliderClient, String cluster, String appId) {
- assert cluster
+ /**
+ * Kill an AM and await restrt
+ * @param sliderClient
+ * @param application
+ * @param appId
+ * @return
+ */
+ public void killAmAndWaitForRestart(String application, String appId) {
+
+ assert application
slider(0, [
- ACTION_AM_SUICIDE, cluster,
+ ACTION_AM_SUICIDE, application,
ARG_EXITCODE, "1",
ARG_WAIT, "1000",
ARG_MESSAGE, "suicide"
@@ -788,9 +795,12 @@ abstract class CommandTestBase extends SliderTestUtils {
sleep(5000)
ensureYarnApplicationIsUp(appId)
- return sliderClient.clusterDescription
}
+ /**
+ * Spinning operation to perform a registry call
+ * @param application application
+ */
protected void ensureRegistryCallSucceeds(String application) {
repeatUntilSuccess(this.&isRegistryAccessible,
REGISTRY_STARTUP_TIMEOUT,
@@ -808,7 +818,10 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
-
+ /**
+ * wait for an application to come up
+ * @param application
+ */
protected void ensureApplicationIsUp(String application) {
repeatUntilSuccess(this.&isApplicationRunning,
SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
@@ -822,6 +835,11 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
protected Outcome isRegistryAccessible(Map<String, String> args) {
String applicationName = args['application'];
SliderShell shell = slider(
@@ -837,11 +855,21 @@ abstract class CommandTestBase extends SliderTestUtils {
return Outcome.fromBool(EXIT_SUCCESS == shell.execute())
}
+ /**
+ * Probe for an application running; uses <code>exists</code> operation
+ * @param args argument map containing <code>"application"</code>
+ * @return
+ */
protected Outcome isApplicationRunning(Map<String, String> args) {
String applicationName = args['application'];
return Outcome.fromBool(isApplicationUp(applicationName))
}
+ /**
+ * Use <code>exists</code> operation to probe for an application being up
+ * @param applicationName app name
+ * @return true if it s running
+ */
protected boolean isApplicationUp(String applicationName) {
return isApplicationInState(
applicationName,
@@ -850,7 +878,8 @@ abstract class CommandTestBase extends SliderTestUtils {
}
/**
- * is an application in a desired yarn state
+ * is an application in a desired yarn state. Uses the <code>exists</code>
+ * CLI operation
* @param yarnState
* @param applicationName
* @return
@@ -863,6 +892,11 @@ abstract class CommandTestBase extends SliderTestUtils {
return shell.ret == 0
}
+ /**
+ * Probe callback for is the the app running or not
+ * @param args map where 'applicationId' must m
+ * @return
+ */
protected Outcome isYarnApplicationRunning(Map<String, String> args) {
String applicationId = args['applicationId'];
@@ -878,12 +912,25 @@ abstract class CommandTestBase extends SliderTestUtils {
*/
public static Outcome isYarnApplicationRunning(
String applicationId) {
- YarnApplicationState appState = lookupYarnAppState(applicationId)
YarnApplicationState yarnState = YarnApplicationState.RUNNING
+ return isYarnApplicationInState(applicationId, yarnState)
+ }
+
+ /**
+ * Probe for a YARN application being in a given state
+ * @param applicationId app id
+ * @param yarnStat desired state
+ * @return success for a match, retry if state below desired, and fail if
+ * above it
+ */
+ public static Outcome isYarnApplicationInState(
+ String applicationId,
+ YarnApplicationState yarnState) {
+ YarnApplicationState appState = lookupYarnAppState(applicationId)
if (yarnState == appState) {
return Outcome.Success;
}
-
+
if (appState.ordinal() > yarnState.ordinal()) {
// app has passed beyond hope
return Outcome.Fail
@@ -891,6 +938,11 @@ abstract class CommandTestBase extends SliderTestUtils {
return Outcome.Retry
}
+ /**
+ * Look up the YARN application by ID, get its application record
+ * @param applicationId the application ID
+ * @return the application state
+ */
public static YarnApplicationState lookupYarnAppState(String applicationId) {
def sar = lookupApplication(applicationId)
assert sar != null;
@@ -898,6 +950,11 @@ abstract class CommandTestBase extends SliderTestUtils {
return appState
}
+ /**
+ * Assert an application is in a given state; fail if not
+ * @param applicationId appId
+ * @param expectedState expected state
+ */
public static void assertInYarnState(String applicationId,
YarnApplicationState expectedState) {
def applicationReport = lookupApplication(applicationId)
@@ -914,14 +971,14 @@ abstract class CommandTestBase extends SliderTestUtils {
ensureYarnApplicationIsUp(id)
return id;
}
+
/**
* Wait for the YARN app to come up. This will fail fast
* @param applicationId
*/
protected void ensureYarnApplicationIsUp(String applicationId) {
repeatUntilSuccess(this.&isYarnApplicationRunning,
- SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
- DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+ instanceLaunchTime,
PROBE_SLEEP_TIME,
[applicationId: applicationId],
true,
@@ -935,6 +992,17 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
+ /**
+ * Get the expected launch time. Default is the configuration option
+ * {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME} and
+ * default value {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME}
+ * @return
+ */
+ public int getInstanceLaunchTime() {
+ return SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS)
+ }
+
public String getInfoAmWebUrl(String applicationName) {
ClusterDescription cd = execStatus(applicationName);
String urlString = cd.getInfo("info.am.web.url");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61e9d1d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index a78ae87..6238164 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -72,9 +72,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
def appId = ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
CONTAINER_LAUNCH_TIMEOUT)
-
- // Wait for 20 secs for AM and agent to both reach STARTED state
- sleep(1000 * 20)
def cd = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
def loggerInstances = cd.instances[COMMAND_LOGGER]
@@ -82,37 +79,32 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
def loggerStats = cd.statistics[COMMAND_LOGGER]
- def origRequested = loggerStats["containers.requested"]
- assert origRequested >= 2
+ assert loggerStats["containers.requested"] == 1
assert loggerStats["containers.live"] == 1
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
- assertSuccess(shell)
-
// Now kill the AM
log.info("Killing AM now ...")
// killAMUsingJsch()
-// killAMUsingAmSuicide()
- killAMUsingVagrantShell()
-
- // Check that the application is not running (and is in ACCEPTED state)
- assert lookupYarnAppState(appId) == YarnApplicationState.ACCEPTED ,
- 'App should be in ACCEPTED state (since AM got killed)'
- log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME)
-
- // Wait until AM comes back up and verify container count again
- ensureYarnApplicationIsUp(appId)
+ killAmAndWaitForRestart(APPLICATION_NAME, appId)
// There should be exactly 1 live logger container
def cd2 = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
// No new containers should be requested for the agents
def loggerStats2 = cd2.statistics[COMMAND_LOGGER]
- assert origRequested == loggerStats2["containers.requested"],
+ assert loggerStats["containers.requested"] == loggerStats2["containers.requested"],
'No new agent containers should be requested'
assert lookupYarnAppState(appId) == YarnApplicationState.RUNNING
}
+ /**
+ * Allow for 2x as long as other test instances, as for AM restart we
+ * need to allow for a longer delay
+ */
+ @Override
+ int getInstanceLaunchTime() {
+ return 2* super.instanceLaunchTime
+ }
protected void killAMUsingAmSuicide() {
SliderShell shell = slider(EXIT_SUCCESS,
@@ -121,12 +113,11 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ARG_MESSAGE, "testAMRestart",
APPLICATION_NAME])
logShell(shell)
- assertSuccess(shell)
}
protected void killAMUsingVagrantShell() {
String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
- assert hostname != null && !hostname.isEmpty()
+ assert hostname
String vagrantVmName = hostname.split("\\.")[0]
String vagrantCwd = sysprop(VAGRANT_CWD)
@@ -146,8 +137,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
protected void killAMUsingJsch() {
String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
String user = UserGroupInformation.currentUser
- assert hostname != null && !hostname.isEmpty()
- assert user != null && !user.isEmpty()
+ assert hostname
+ assert user
bindSSHKey()
RemoteServer remoteServer = new RemoteServer(
[3/4] git commit: SLIDER-616 AppsThroughAgentIT working;
improved assertion logic
Posted by st...@apache.org.
SLIDER-616 AppsThroughAgentIT working; improved assertion logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2b8dd550
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2b8dd550
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2b8dd550
Branch: refs/heads/develop
Commit: 2b8dd5502656258c43598fbebbc2bad7e76c9399
Parents: d64b2c7
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 6 14:39:26 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 6 14:39:26 2014 +0000
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 70 ++++++++++++++++++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 75 +-------------------
2 files changed, 72 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b8dd550/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 1d436da..210e257 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -1187,4 +1187,74 @@ abstract class CommandTestBase extends SliderTestUtils {
assertContainersLive(application, component, expected)
}
}
+
+ /**
+ * Spin for <code>REGISTRY_STARTUP_TIMEOUT</code> waiting
+ * for the output of the registry command to contain the specified
+ * values
+ * @param outfile file to save the output to
+ * @param commands commands to execute
+ * @param match text to match on
+ */
+ public void awaitRegistryOutfileContains(
+ File outfile,
+ List<String> commands,
+ String match) {
+ def errorText = "failed to find $match in output. Check your hadoop versions and the agent logs"
+ repeatUntilSuccess("registry",
+ this.&generatedFileContains,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
+ [
+ text : match,
+ filename: outfile.absolutePath,
+ command : commands
+ ],
+ true,
+ errorText) {
+ slider(0, commands).dumpOutput()
+ if (!outfile.length()) {
+ log.warn("No exported entries.\n" +
+ "Is your application using a consistent version of Hadoop? ")
+ }
+ fail(errorText + "\n" + outfile.text)
+ }
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandOutputContains(Map args) {
+ String text = args['text'];
+ List<String> command = (List < String >)args['command']
+ SliderShell shell = slider(0, command)
+ return Outcome.fromBool(shell.outputContains(text))
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandSucceeds(Map args) {
+ List<String> command = (List<String>) args['command']
+ SliderShell shell = slider(command)
+ return Outcome.fromBool(shell.ret == 0)
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map
+ * @return probe outcome
+ */
+ protected Outcome generatedFileContains(Map args) {
+ List<String> command = (List<String>) args['command']
+ String text = args['text'];
+ String filename = args['filename'];
+ File f = new File(filename)
+ f.delete()
+ SliderShell shell = slider(0, command)
+ shell.dumpOutput()
+ assert f.exists()
+ return Outcome.fromBool(f.text.contains(text))
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b8dd550/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 8ddc38f..8b15a0c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -26,7 +26,6 @@ import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.test.Outcome
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -90,9 +89,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
log.info(amWebUrl.toURL().text);
ensureRegistryCallSucceeds(application)
- def outfile = tmpFile(".txt")
-
assertAppRunning(appId)
+ def outfile = tmpFile(".txt")
def commands = [
ACTION_REGISTRY,
@@ -105,15 +103,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
awaitRegistryOutfileContains(outfile, commands, "container_log_dirs")
awaitRegistryOutfileContains(outfile, commands, "container_work_dirs")
- // get log folders
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_REGISTRY,
- ARG_NAME,
- application,
- ARG_LISTEXP])
- assertOutputContains(shell, "container_log_dirs")
- assertOutputContains(shell, "container_work_dirs")
// get log folders
shell = slider(EXIT_SUCCESS,
@@ -125,7 +114,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
"container_log_dirs"])
assertOutputContains(shell, '"tag" : "COMMAND_LOGGER"', 2)
- assertOutputContains(shell, '"level" : "level"', 2)
+ assertOutputContains(shell, '"level" : "component"', 2)
// get cl-site config
@@ -157,64 +146,4 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assertAppRunning(appId)
}
- public awaitRegistryOutfileContains(
- File outfile,
- List<String> commands,
- String match) {
- repeatUntilSuccess("registry",
- this.&generatedFileContains,
- REGISTRY_STARTUP_TIMEOUT * 2,
- PROBE_SLEEP_TIME,
- [
- text : match,
- filename: outfile.absolutePath,
- command : commands
- ],
- true,
- "failed to find $match in output") {
- slider(0, commands).dumpOutput()
- fail("no $match in \n$outfile.text")
- }
- }
-
- /**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
- * @return probe outcome
- */
- protected Outcome commandOutputContains(Map args) {
- String text = args['text'];
- List<String> command = (List < String >)args['command']
- SliderShell shell = slider(0, command)
- return Outcome.fromBool(shell.outputContains(text))
- }
-
- /**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
- * @return probe outcome
- */
- protected Outcome commandSucceeds(Map args) {
- List<String> command = (List<String>) args['command']
- SliderShell shell = slider(command)
- return Outcome.fromBool(shell.ret == 0)
- }
-
- /**
- * Is the registry accessible for an application?
- * @param args argument map
- * @return probe outcome
- */
- protected Outcome generatedFileContains(Map args) {
- List<String> command = (List<String>) args['command']
- String text = args['text'];
- String filename = args['filename'];
- File f = new File(filename)
- f.delete()
- SliderShell shell = slider(0, command)
- shell.dumpOutput()
- assert f.exists()
- return Outcome.fromBool(f.text.contains(text))
- }
-
}