You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/21 15:57:40 UTC

[6/9] git commit: SLIDER-548 Slider exists command to add --state probe

SLIDER-548 Slider exists command to add --state probe


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee1c2447
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee1c2447
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee1c2447

Branch: refs/heads/develop
Commit: ee1c24470efcd3c1793b86a3a35daaf6f409564c
Parents: c0b2f76
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 14:22:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 21 14:22:15 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  |  61 +++++---
 .../slider/common/params/ActionExistsArgs.java  |   7 +-
 .../apache/slider/common/params/Arguments.java  |   1 +
 .../agent/actions/TestActionExists.groovy       |   8 +-
 .../funtest/framework/CommandTestBase.groovy    |  32 +++--
 .../funtest/lifecycle/AMFailuresIT.groovy       |   8 +-
 .../funtest/lifecycle/AgentFailures2IT.groovy   |   2 +-
 .../funtest/lifecycle/AgentFailuresIT.groovy    |   2 +-
 .../funtest/lifecycle/AgentRegistryIT.groovy    | 141 +++++++++++++++++++
 .../funtest/lifecycle/AppsThroughAgentIT.groovy |   2 +-
 .../AppsThroughAgentQueueAndLabelsIT.groovy     |   2 +-
 11 files changed, 222 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5482566..d503637 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -66,6 +66,7 @@ import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.params.AbstractActionArgs;
 import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
 import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionExistsArgs;
 import org.apache.slider.common.params.ActionInstallKeytabArgs;
 import org.apache.slider.common.params.ActionInstallPackageArgs;
 import org.apache.slider.common.params.ActionAMSuicideArgs;
@@ -154,6 +155,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -373,7 +375,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
       exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
     } else if (ACTION_EXISTS.equals(action)) {
       exitCode = actionExists(clusterName,
-          serviceArgs.getActionExistsArgs().live);
+          serviceArgs.getActionExistsArgs());
     } else if (ACTION_FLEX.equals(action)) {
       exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
     } else if (ACTION_HELP.equals(action)) {
@@ -1724,10 +1726,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   }
 
   @Override
-  @VisibleForTesting
   public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
+    ActionExistsArgs args = new ActionExistsArgs();
+    args.live = checkLive;
+    return actionExists(name, args);
+  }
+
+  public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
     verifyBindingsDefined();
     SliderUtils.validateClusterName(name);
+    boolean checkLive = args.live;
     log.debug("actionExists({}, {})", name, checkLive);
 
     //initial probe for a cluster in the filesystem
@@ -1735,31 +1743,40 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) {
       throw unknownClusterException(name);
     }
+    String state = args.state;
+    if (!checkLive && SliderUtils.isUnset(state)) {
+      log.info("Application {} exists", name);
+      return EXIT_SUCCESS;
+    }
     
-    //test for liveness if desired
-
-    if (checkLive) {
-      ApplicationReport instance = findInstance(name);
-      if (instance == null) {
-        log.info("Cluster {} not running", name);
-        return EXIT_FALSE;
-      } else {
-        // the app exists, but it may be in a terminated state
-        SliderUtils.OnDemandReportStringifier report =
+    //test for liveness/state
+    ApplicationReport instance = findInstance(name);
+    if (instance == null) {
+      log.info("Application {} not running", name);
+      return EXIT_FALSE;
+    } else {
+      // the app exists, but it may be in a terminated state
+      SliderUtils.OnDemandReportStringifier report =
           new SliderUtils.OnDemandReportStringifier(instance);
-        YarnApplicationState state =
+      YarnApplicationState appstate =
           instance.getYarnApplicationState();
-        if (state.ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
-          //cluster in the list of apps but not running
-          log.info("Cluster {} found but is in state {}", name, state);
-          log.debug("State {}", report);
-          return EXIT_FALSE;
-        }
-        log.info("Cluster {} is live:\n{}", name, report);
+      boolean inDesiredState;
+      if (checkLive) {
+        inDesiredState =
+            appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
+      } else {
+        state = state.toUpperCase(Locale.ENGLISH);
+        inDesiredState = state.equals(appstate.toString());
       }
-    } else {
-      log.info("Cluster {} exists", name);
+      if (!inDesiredState) {
+        //cluster in the list of apps but not running
+        log.info("Application {} found but is in wrong state {}", name, appstate);
+        log.debug("State {}", report);
+        return EXIT_FALSE;
+      }
+      log.info("Application {} is {}\n{}", name, appstate, report);
     }
+
     return EXIT_SUCCESS;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
index 2d6bef2..047d32c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
@@ -31,6 +31,11 @@ public class ActionExistsArgs extends AbstractActionArgs {
     return SliderActions.ACTION_EXISTS;
   }
   @Parameter(names = {ARG_LIVE},
-             description = "verify that the cluster is running")
+             description = "verify that the application is running")
   public boolean live;
+  
+  @Parameter(names = {ARG_STATE},
+             description = "verify that the application is in the specific YARN state")
+  public String state = "";
+  
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 2f7af70..06d9dfb 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -88,6 +88,7 @@ public interface Arguments {
   String ARG_RESOURCE_OPT_SHORT = "-ro";
   String ARG_SERVICETYPE = "--servicetype";
   String ARG_SLIDER = "--slider";
+  String ARG_STATE = "--state";
   String ARG_SYSPROP = "-S";
   String ARG_TEMPLATE = "--template";
   String ARG_USER = "--user";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index a190b7d..2645e27 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -21,9 +21,11 @@ package org.apache.slider.agent.actions
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.agent.AgentMiniClusterTestBase
 import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionExistsArgs
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
 import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
@@ -107,8 +109,12 @@ class TestActionExists extends AgentMiniClusterTestBase {
     assertSucceeded(launcher)
     
     // assert that the cluster exists
-
     assert 0 == sliderClient.actionExists(clustername, true)
+
+    // assert that the cluster is in the running state
+    ActionExistsArgs args = new ActionExistsArgs()
+    args.state = YarnApplicationState.RUNNING.toString()
+    assert 0 == sliderClient.actionExists(clustername, args)
     
     // stop the cluster
     clusterActionFreeze(sliderClient, clustername)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 5f3fd9e..1878f63 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -25,6 +25,8 @@ import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hdfs.HdfsConfiguration
 import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.util.Shell
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.common.tools.ConfigHelper
 import org.apache.slider.core.main.LauncherExitCodes
@@ -646,9 +648,9 @@ abstract class CommandTestBase extends SliderTestUtils {
 
   protected void ensureApplicationIsUp(String clusterName) {
     repeatUntilTrue(this.&isApplicationUp,
-        20,
-        1000 * SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+        SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
             DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+        1000,
         ['arg1': clusterName],
         true,
         'Application did not start, aborting test.')
@@ -656,20 +658,24 @@ abstract class CommandTestBase extends SliderTestUtils {
 
   protected boolean isApplicationUp(Map<String, String> args) {
     String applicationName = args['arg1'];
-    return isApplicationInState("RUNNING", applicationName);
+    return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
   }
 
-  public static boolean isApplicationInState(String text, String applicationName) {
-    boolean exists = false
-    SliderShell shell = slider(0,
-      [ACTION_LIST, applicationName])
-    for (String str in shell.out) {
-      if (str.contains(text)) {
-        exists = true
-      }
-    }
+  protected boolean isApplicationUp(String applicationName) {
+    return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
+  }
+
+  /**
+   * 
+   * @param yarnState
+   * @param applicationName
+   * @return
+   */
+  public static boolean isApplicationInState(YarnApplicationState yarnState, String applicationName) {
+    SliderShell shell = slider(
+      [ACTION_EXISTS, applicationName, ARG_STATE, yarnState.toString()])
 
-    return exists
+    return shell.ret == 0
   }
 
   protected void repeatUntilTrue(Closure c, int maxAttempts, int sleepDur, Map args,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 988d34d..159e2d6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -25,6 +25,7 @@ import org.apache.bigtop.itest.shell.Shell
 import org.apache.chaos.remote.RemoteServer
 import org.apache.chaos.remote.SshCommands
 import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.params.Arguments
@@ -79,7 +80,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     assert live != null && live.isInteger() && live.toInteger() == 1,
         'At least 1 container must be live now'
 
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
     assertSuccess(shell)
 
     // Now kill the AM
@@ -89,7 +90,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     killAMUsingVagrantShell()
 
     // Check that the application is not running (and is in ACCEPTED state)
-    assert isApplicationInState("ACCEPTED", APPLICATION_NAME), 
+    assert isApplicationInState(YarnApplicationState.ACCEPTED,
+        APPLICATION_NAME), 
       'App should be in ACCEPTED state (since AM got killed)'
     log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME)
 
@@ -113,7 +115,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
         'No new agent containers should be requested'
     log.info("After AM KILL: no new agent containers were requested")
 
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
     assertSuccess(shell)
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 8a4140c..a5cf5be 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -74,7 +74,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
         'At least 2 containers must be requested'
 
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
 
     assertSuccess(shell)
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 7575fc6..a384170 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -75,7 +75,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     assert requested != null && requested.isInteger() && requested.toInteger() >= 2,
         'At least 2 containers must be requested'
 
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
 
     assertSuccess(shell)
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
new file mode 100644
index 0000000..2df5f56
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.Endpoint
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentRegistryIT extends AgentCommandTestBase
+    implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+  static String CLUSTER = "test-agent-registry"
+
+  static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+
+
+  @Before
+  public void prepareCluster() {
+    setupCluster(CLUSTER)
+    describe("Create a 0-role cluster and make registry queries against it")
+
+  }
+
+  @After
+  public void destroyCluster() {
+    cleanup(CLUSTER)
+  }
+
+  @Test
+  public void testAgentClusterLifecycle() throws Throwable {
+
+    describe "Walk a 0-role cluster through its lifecycle"
+
+    // sanity check to verify the config is correct
+    assert clusterFS.uri.scheme != "file"
+
+    def clusterpath = buildClusterPath(CLUSTER)
+    assert !clusterFS.exists(clusterpath)
+    SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+        APP_TEMPLATE,
+        APP_RESOURCE2)
+
+    logShell(shell)
+
+    ensureApplicationIsUp(CLUSTER)
+
+    //at this point the cluster should exist.
+    assertPathExists(
+        clusterFS,
+        "Cluster parent directory does not exist",
+        clusterpath.parent)
+
+    assertPathExists(clusterFS, "Cluster directory does not exist", clusterpath)
+
+    // resolve the ~ path
+
+    resolve(0, [ARG_LIST, ARG_PATH, "/"])
+    resolve(0, [ARG_LIST, ARG_PATH, "/users"])
+
+    resolve(0, [ARG_LIST, ARG_PATH, TILDE]).dumpOutput()
+
+
+    String sliderApps = "${TILDE}/services/${SliderKeys.APP_TYPE}"
+    resolve(0, [ARG_LIST, ARG_PATH, sliderApps]).dumpOutput()
+
+    // running app
+    String appPath = sliderApps +"/"+ CLUSTER
+    resolve(0, [ARG_LIST, ARG_PATH, appPath]).dumpOutput()
+
+    resolve(0, [ARG_PATH, appPath]).dumpOutput()
+    // and the service record
+    File serviceRecordFile = File.createTempFile("tempfile", ".json")
+    serviceRecordFile.deleteOnExit()
+    resolve(0, [ARG_PATH, appPath,
+                ARG_OUTPUT, serviceRecordFile.absolutePath])
+    RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal()
+
+    ServiceRecord serviceRecord = marshal.fromFile(serviceRecordFile)
+    def ipcEndpoint = serviceRecord.external.find { Endpoint epr ->
+      epr.api == AM_IPC_PROTOCOL;
+    }
+    assert ipcEndpoint != null;
+    def endpoints = [:]
+    serviceRecord.external.each { Endpoint epr ->
+      endpoints[epr.api] = epr;
+    }
+    serviceRecord.internal.each { Endpoint epr ->
+      endpoints[epr.api] = epr;
+    }
+    assert endpoints[PUBLISHER_REST_API]
+    assert endpoints[REGISTRY_REST_API]
+    assert endpoints[AGENT_SECURE_REST_API]
+    assert endpoints[AGENT_ONEWAY_REST_API]
+
+    //stop
+    freeze(0, CLUSTER,
+        [
+            ARG_FORCE,
+            ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+            ARG_MESSAGE, "final-shutdown"
+        ])
+
+    destroy(0, CLUSTER)
+
+    //cluster now missing
+    exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index ff55b05..9389f80 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -149,6 +149,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
       assert fail("Should have exported cl-site")
     }
 
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee1c2447/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index 6732691..c87d507 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -115,7 +115,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     assertComponentCount(COMMAND_LOGGER, 2, shell)
 
     assertSuccess(shell)
-    assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
   }
 
   boolean hasContainerCountExceeded(Map<String, String> args) {