You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/02 15:25:43 UTC

[31/50] git commit: SLIDER-570: tuning of tests, agent failure tests working (i.e. regression fixed)

SLIDER-570: tuning of tests, agent failure tests working (i.e. regression fixed)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1023ead
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1023ead
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1023ead

Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: e1023eadf896866ce6c63f06e47bf835adc8576c
Parents: 6e171c7
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 13:25:26 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 13:25:26 2014 +0000

----------------------------------------------------------------------
 .../framework/AgentCommandTestBase.groovy       | 10 ++++
 .../funtest/framework/CommandTestBase.groovy    | 54 ++++++++++++--------
 .../funtest/lifecycle/AMFailuresIT.groovy       |  3 +-
 .../lifecycle/AgentClusterLifecycleIT.groovy    |  1 +
 .../funtest/lifecycle/AgentFailures2IT.groovy   | 12 ++---
 .../funtest/lifecycle/AgentFailuresIT.groovy    |  8 +--
 .../lifecycle/AgentLaunchFailureIT.groovy       |  3 +-
 .../funtest/lifecycle/AgentRegistryIT.groovy    |  2 -
 .../lifecycle/AppsThroughAgentDemo.groovy       | 27 ++++++----
 .../funtest/lifecycle/AppsThroughAgentIT.groovy | 16 +++---
 .../AppsThroughAgentQueueAndLabelsIT.groovy     |  6 ++-
 11 files changed, 84 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 6c99ab6..6e05056 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
@@ -167,4 +168,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
       assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}")
     }
   }
+
+  /**
+   * Assert that the application is running (i.e in state
+   * {@link YarnApplicationState#RUNNING})
+   * @param appId application ID
+   */
+  def assertAppRunning(String appId) {
+    assertInYarnState(appId, YarnApplicationState.RUNNING)
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index fc74129..7ea84df 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -94,8 +94,7 @@ abstract class CommandTestBase extends SliderTestUtils {
   public static final int CONTAINER_LAUNCH_TIMEOUT = 90000
   public static final int PROBE_SLEEP_TIME = 4000
   public static final int REGISTRY_STARTUP_TIMEOUT = 60000
-  public static
-  final String E_LAUNCH_FAIL = 'Application did not start'
+  public static final String E_LAUNCH_FAIL = 'Application did not start'
 
   /*
   Static initializer for test configurations. If this code throws exceptions
@@ -380,14 +379,18 @@ abstract class CommandTestBase extends SliderTestUtils {
   static SliderShell lookup(int result, String id, File out) {
     assert id
     def commands = [ACTION_LOOKUP, ARG_ID, id]
-    if (out) commands += [ARG_OUTPUT, out.absolutePath]
+    if (out) {
+      commands += [ARG_OUTPUT, out.absolutePath]
+    }
     slider(result, commands)
   }
   
   static SliderShell lookup(String id, File out) {
     assert id
     def commands = [ACTION_LOOKUP, ARG_ID, id]
-    if (out) commands += [ARG_OUTPUT, out.absolutePath]
+    if (out) {
+      commands += [ARG_OUTPUT, out.absolutePath]
+    }
     slider(commands)
   }
 
@@ -707,7 +710,7 @@ abstract class CommandTestBase extends SliderTestUtils {
     }
     return args
   }
-  
+
   public static SerializedApplicationReport maybeLoadAppReport(File reportFile) {
     if (reportFile.exists() && reportFile.length()> 0) {
       ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
@@ -776,19 +779,24 @@ abstract class CommandTestBase extends SliderTestUtils {
         ARG_MESSAGE, "suicide"
     ])
 
-
     sleep(5000)
     ensureApplicationIsUp(cluster)
-    
-/*
-    def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
-        DEFAULT_AM_RESTART_SLEEP_TIME)
-    sleep(sleeptime)
-*/
-    ClusterDescription status
+    return sliderClient.clusterDescription
+  }
+  public ClusterDescription killAmAndWaitForRestart(
+      SliderClient sliderClient, String cluster, String appId) {
+
+    assert cluster
+    slider(0, [
+        ACTION_AM_SUICIDE, cluster,
+        ARG_EXITCODE, "1",
+        ARG_WAIT, "1000",
+        ARG_MESSAGE, "suicide"
+    ])
 
-    status = sliderClient.clusterDescription
-    return status
+    sleep(5000)
+    ensureYarnApplicationIsUp(appId)
+    return sliderClient.clusterDescription
   }
 
   protected void ensureRegistryCallSucceeds(String application) {
@@ -844,7 +852,6 @@ abstract class CommandTestBase extends SliderTestUtils {
     );
   }
 
-  
   /**
    * is an application in a desired yarn state 
    * @param yarnState
@@ -1057,25 +1064,30 @@ abstract class CommandTestBase extends SliderTestUtils {
     int expectedCount = args['limit'].toInteger();
 
     int requestedCount = queryRequestedCount(application, role)
-    log.debug("requested count = $requestedCount; expected=$expectedCount")
+    log.debug("requested $role count = $requestedCount; expected=$expectedCount")
     return Outcome.fromBool(requestedCount >= expectedCount)
   }
 
-  void expectContainerRequestedCountReached(String application, String role, int limit) {
+  void expectContainerRequestedCountReached(String application, String role, int limit,
+      int container_launch_timeout) {
 
     repeatUntilSuccess(
         this.&hasRequestedContainerCountReached,
-        CONTAINER_LAUNCH_TIMEOUT,
+        container_launch_timeout,
         PROBE_SLEEP_TIME,
         [limit      : Integer.toString(limit),
          role       : role,
          application: application],
         true,
         "countainer count not reached") {
-      describe "container count not reached"
+      int requestedCount = queryRequestedCount(application, role)
+
+      def message = "expected count of $role = $limit not reached: $requestedCount" +
+                    " after $container_launch_timeout mS"
+      describe message
       ClusterDescription cd = execStatus(application);
       log.info("Parsed status \n$cd")
-      status(application).dumpOutput()
+      fail(message)
     };
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 2e28c84..7cc01b8 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -70,7 +70,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     logShell(shell)
 
     def appId = ensureYarnApplicationIsUp(launchReportFile)
-    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1)
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+        CONTAINER_LAUNCH_TIMEOUT)
     
     // Wait for 20 secs for AM and agent to both reach STARTED state
     sleep(1000 * 20)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 3e5cec7..cb137ce 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -183,6 +183,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
 
       //start with a restart count set to enable restart
       describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+      
       thaw(CLUSTER,
           [
               ARG_WAIT, Integer.toString(THAW_WAIT_TIME),

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 6c6b52b..bfae6ec 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
@@ -60,13 +59,12 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     logShell(shell)
 
     def appId = ensureYarnApplicationIsUp(launchReportFile)
-    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+        CONTAINER_LAUNCH_TIMEOUT * 2)
     sleep(1000 * 20)
-    def cd = execStatus(APPLICATION_NAME)
-    assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
-    assertInYarnState(appId, YarnApplicationState.RUNNING)
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+        CONTAINER_LAUNCH_TIMEOUT * 2)
+    assertAppRunning(appId)
   }
 
-
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 0410881..efd6194 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -63,13 +63,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     logShell(shell)
 
     def appId = ensureYarnApplicationIsUp(launchReportFile)
-
-    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2)
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+        CONTAINER_LAUNCH_TIMEOUT)
     sleep(1000 * 20)
-    assertInYarnState(appId, YarnApplicationState.RUNNING)
+    assertAppRunning(appId)
     def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
     assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
-    assertInYarnState(appId, YarnApplicationState.RUNNING)
+    assertAppRunning(appId)
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index 5087c06..791123c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -28,7 +28,6 @@ import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
 import org.apache.slider.funtest.framework.AgentCommandTestBase
 import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.SliderShell
 import org.junit.After
 import org.junit.Before
 import org.junit.Test
@@ -62,7 +61,7 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
  
     // create an AM which fails to launch
     File launchReportFile = createAppReportFile();
-    SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+    createTemplatedSliderApplication(CLUSTER,
         APP_TEMPLATE,
         APP_RESOURCE2,
         [

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index b5ee23d..5b8b3cd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -132,7 +132,5 @@ public class AgentRegistryIT extends AgentCommandTestBase
 
     //cluster now missing
     exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
-
-
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
index eb825fc..5be7211 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
@@ -49,13 +49,17 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     assumeAgentTestsEnabled()
 
     cleanup(APPLICATION_NAME)
+    File launchReportFile = createAppReportFile();
+
     SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
         APP_TEMPLATE,
-        APP_RESOURCE)
+        APP_RESOURCE,
+        [],
+        launchReportFile)
 
     logShell(shell)
 
-    ensureApplicationIsUp(APPLICATION_NAME)
+    def appId = ensureYarnApplicationIsUp(launchReportFile)
 
     //flex
     slider(EXIT_SUCCESS,
@@ -69,9 +73,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     // sleep till the new instance starts
     sleep(1000 * 10)
 
-    shell = slider(EXIT_SUCCESS,
-        [
-            ACTION_STATUS,
+    slider(EXIT_SUCCESS,
+        [ACTION_STATUS,
             APPLICATION_NAME])
 
     expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
@@ -88,7 +91,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
             ACTION_REGISTRY,
             ARG_NAME,
             APPLICATION_NAME,
-            ARG_LISTEXP])
+            ARG_LISTEXP
+        ])
     if(!containsString(shell, "container_log_dirs") ||
        !containsString(shell, "container_work_dirs")) {
       logShell(shell)
@@ -102,7 +106,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
             ARG_NAME,
             APPLICATION_NAME,
             ARG_GETEXP,
-            "container_log_dirs"])
+            "container_log_dirs"
+        ])
     if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
     || !containsString(shell, "\"level\" : \"component\"", 2)) {
       logShell(shell)
@@ -116,7 +121,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
             ARG_NAME,
             APPLICATION_NAME,
             ARG_GETEXP,
-            "container_work_dirs"])
+            "container_work_dirs"
+        ])
     if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
     || !containsString(shell, "\"level\" : \"component\"", 2)) {
       logShell(shell)
@@ -132,7 +138,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
             ARG_GETCONF,
             "cl-site",
             ARG_FORMAT,
-            "json"])
+            "json"
+        ])
 
     for (int i = 0; i < 10; i++) {
       if (shell.getRet() != EXIT_SUCCESS) {
@@ -155,6 +162,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
       assert fail("Should have exported cl-site")
     }
 
-    assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+    assertAppRunning(appId)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 5a5b964..75807c3 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
@@ -76,10 +75,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
     // sleep till the new instance starts
     sleep(1000 * 10)
 
-    shell = slider(EXIT_SUCCESS,
-        [
-            ACTION_STATUS,
-            APPLICATION_NAME])
+    status(0, APPLICATION_NAME)
 
     expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
 
@@ -145,7 +141,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
       if (shell.getRet() != EXIT_SUCCESS) {
         println "Waiting for the cl-site to show up"
         sleep(1000 * 10)
-        shell = slider(
+        shell = slider(0,
             [
                 ACTION_REGISTRY,
                 ARG_NAME,
@@ -156,12 +152,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
                 "json"])
       }
     }
-    assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved"
     if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) {
       logShell(shell)
-      assert fail("Should have exported cl-site")
+      
+      fail("Should have exported cl-site; got " +
+                  "stdout"  +shell.stdErrHistory +
+                  " \nstderr:" + shell.stdErrHistory)
     }
 
-    assertInYarnState(appId,  YarnApplicationState.RUNNING)
+    assertAppRunning(appId)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index ee418dc..f6a1b1e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -92,7 +92,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
     def appId = ensureYarnApplicationIsUp(launchReportFile)
 
-    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1 )
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+        CONTAINER_LAUNCH_TIMEOUT)
     expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
 
     //flex
@@ -107,7 +108,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
     // spin till the flexed instance starts
     ensureYarnApplicationIsUp(appId)
-    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
+    expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+        CONTAINER_LAUNCH_TIMEOUT)
 
 
     sleep(1000 * 20)