You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/23 18:20:50 UTC
[38/50] incubator-slider git commit: SLIDER-970: AASleepIT,
test enums cluster size, asks for one more, verifies that it's not there
SLIDER-970: AASleepIT, test enums cluster size, asks for one more, verifies that it's not there
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0c1977be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0c1977be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0c1977be
Branch: refs/heads/develop
Commit: 0c1977be6be92962235a12d09745d95360be09c0
Parents: eac0de9
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 20 17:49:27 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 20 17:49:27 2015 +0000
----------------------------------------------------------------------
.../test_min_pkg/sleep_cmd/resources.json | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 23 ++++
.../funtest/framework/CommandTestBase.groovy | 110 ++++++++++++-------
.../ApplicationWithAddonPackagesIT.groovy | 19 ----
.../slider/funtest/lifecycle/AASleepIT.groovy | 59 ++++++----
.../funtest/lifecycle/AgentWebPagesIT.groovy | 1 -
6 files changed, 132 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
index 1268996..e0ed16a 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
@@ -17,7 +17,7 @@
"yarn.role.priority": "2",
"yarn.component.instances": "0",
"yarn.memory": "128",
- "yarn.placement.policy": "4"
+ "yarn.component.placement.policy": "4"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 5ef388a..cb6ce0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -884,6 +884,29 @@ class SliderTestUtils extends Assert {
log.info("$realkey = $val")
return val
}
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() {
+ return tmpFile(".json")
+ }
+
+ /**
+ * Create a temp file with the specific name. It's deleted after creation,
+ * to avoid "file exists exceptions"
+ * @param suffix suffix, e.g. ".txt"
+ * @return a path to a file which may be created
+ */
+ public static File tmpFile(String suffix) {
+ File reportFile = File.createTempFile(
+ "temp",
+ suffix,
+ new File("target"))
+ reportFile.delete()
+ return reportFile
+ }
/**
* Execute a closure, assert it fails with a given exit code and text
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 654ccb3..252bb79 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -29,6 +29,7 @@ import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
+import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
@@ -38,6 +39,7 @@ import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.launch.SerializedApplicationReport
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.ApplicationReportSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.test.SliderTestUtils
import org.apache.slider.test.Outcome;
@@ -206,7 +208,7 @@ abstract class CommandTestBase extends SliderTestUtils {
return false;
}
}
-
+
/**
* Add a jar to the slider classpath by looking up a class and determining
* its containing JAR
@@ -325,7 +327,6 @@ abstract class CommandTestBase extends SliderTestUtils {
])
}
-
static SliderShell freeze(
int exitCode,
String name,
@@ -359,9 +360,9 @@ abstract class CommandTestBase extends SliderTestUtils {
static SliderShell killContainer(String name, String containerID) {
slider(0,
[
- ACTION_KILL_CONTAINER,
- name,
- containerID
+ ACTION_KILL_CONTAINER,
+ name,
+ containerID
])
}
@@ -440,9 +441,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
static SliderShell registry(Collection<String> commands) {
- slider(0,
- [ACTION_REGISTRY] + commands
- )
+ slider(0, [ACTION_REGISTRY] + commands)
}
/**
@@ -572,7 +571,7 @@ abstract class CommandTestBase extends SliderTestUtils {
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
- SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+ SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
if (blockUntilRunning) {
@@ -774,24 +773,6 @@ abstract class CommandTestBase extends SliderTestUtils {
}
/**
- * Create a temp JSON file. After coming up with the name, the file
- * is deleted
- * @return the filename
- */
- public static File createTempJsonFile() {
- return tmpFile(".json")
- }
-
- public static File tmpFile(String suffix) {
- File reportFile = File.createTempFile(
- "launch",
- suffix,
- new File("target"))
- reportFile.delete()
- return reportFile
- }
-
- /**
* If the option is not null/empty, add the command and the option
* @param args arg list being built up
* @param command command to add option
@@ -812,20 +793,20 @@ abstract class CommandTestBase extends SliderTestUtils {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- }
+ }
return null;
- }
-
+ }
+
public static SerializedApplicationReport loadAppReport(File reportFile) {
if (reportFile.exists() && reportFile.length() > 0) {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- } else {
+ }else {
throw new FileNotFoundException(reportFile.absolutePath)
- }
- }
-
+ }
+ }
+
public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
def report = maybeLoadAppReport(launchReport)
if (report) {
@@ -856,7 +837,44 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
-
+ /**
+ * Lookup an application, return null if loading failed
+ * @param id application ID
+ * @return an application report or null
+ */
+ public static NodeInformationList listNodes(boolean healthy = false, String label = "") {
+ File reportFile = createTempJsonFile();
+ try {
+ def shell = nodes(reportFile, healthy, label)
+ shell.dumpOutput()
+ JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer();
+ serDeser.fromFile(reportFile)
+ } finally {
+ reportFile.delete()
+ }
+ }
+
+ /**
+ * List cluster nodes
+ * @param out output file (or null)
+ * @param healthy list healthy nodes only
+ * @param label label to filter on
+ * @return output
+ */
+ static SliderShell nodes(File out, boolean healthy = false, String label = "") {
+ def commands = [ACTION_NODES]
+ if (label) {
+ commands += [ ARG_LABEL, label]
+ }
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
+ if (healthy) {
+ commands << ARG_HEALTHY
+ }
+ slider(0, commands)
+ }
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
@@ -1274,6 +1292,13 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
+ /**
+ * Assert that exactly the number of containers are live
+ * @param clustername name of cluster
+ * @param component component to probe
+ * @param count count
+ * @return
+ */
public ClusterDescription assertContainersLive(String clustername,
String component,
int count) {
@@ -1420,8 +1445,8 @@ abstract class CommandTestBase extends SliderTestUtils {
}
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for the output {@code command: List} containing {@code text}
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandOutputContains(Map args) {
@@ -1430,9 +1455,10 @@ abstract class CommandTestBase extends SliderTestUtils {
SliderShell shell = slider(0, command)
return Outcome.fromBool(shell.outputContains(text))
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for a command {@code command: List} succeeeding
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandSucceeds(Map args) {
@@ -1440,9 +1466,11 @@ abstract class CommandTestBase extends SliderTestUtils {
SliderShell shell = slider(command)
return Outcome.fromBool(shell.ret == 0)
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map
+ * probe for a command {@code command: List} generating a file 'filename'
+ * which must contain the text 'text'
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome generatedFileContains(Map args) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
index 43275e6..b32275e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
@@ -18,28 +18,9 @@ package org.apache.slider.funtest.coprocessors
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.funtest.framework.FileUploader
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.After
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
index 464b329..054245e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
@@ -21,7 +21,9 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -40,7 +42,7 @@ public class AASleepIT extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
- static String CLUSTER = "test-aa-sleep"
+ static String NAME = "test-aa-sleep"
static String TEST_RESOURCE = ResourcePaths.SLEEP_RESOURCES
static String TEST_METADATA = ResourcePaths.SLEEP_META
@@ -49,23 +51,36 @@ public class AASleepIT extends AgentCommandTestBase
@Before
public void prepareCluster() {
- setupCluster(CLUSTER)
+ setupCluster(NAME)
}
@After
public void destroyCluster() {
- cleanup(CLUSTER)
+ cleanup(NAME)
}
@Test
public void testAASleepIt() throws Throwable {
describe("Test Anti-Affinity Placement")
- def clusterpath = buildClusterPath(CLUSTER)
+
+ describe "diagnostics"
+
+ slider([ACTION_DIAGNOSTICS, ARG_VERBOSE, ARG_CLIENT, ARG_YARN, ARG_CREDENTIALS])
+
+ describe "list nodes"
+
+ def healthyNodes = listNodes(true)
+
+ def healthyNodeCount = healthyNodes.size()
+ describe("Cluster nodes : ${healthyNodeCount}")
+ log.info(NodeInformationList.createSerializer().toJson(healthyNodes))
+
File launchReportFile = createTempJsonFile();
- // TODO: Determine YARN cluster size via an API/CLI Call, maybe use labels too?
- int desired = buildDesiredCount(1)
- SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
+ int desired = buildDesiredCount(healthyNodeCount)
+ def clusterpath = buildClusterPath(NAME)
+
+ SliderShell shell = createSliderApplicationMinPkg(NAME,
TEST_METADATA,
TEST_RESOURCE,
ResourcePaths.SLEEP_APPCONFIG,
@@ -84,30 +99,26 @@ public class AASleepIT extends AgentCommandTestBase
assertPathExists(clusterFS, "Cluster directory does not exist", clusterpath)
- status(0, CLUSTER)
+ status(0, NAME)
def expected = buildExpectedCount(desired)
- expectLiveContainerCountReached(CLUSTER, SLEEP_100, expected,
+ expectLiveContainerCountReached(NAME, SLEEP_100, expected,
CONTAINER_LAUNCH_TIMEOUT)
- operations(CLUSTER, loadAppReport(launchReportFile), desired, expected)
-
- // sleep for some manual test
- describe("You may quickly perform manual tests against the application instance $CLUSTER")
- sleep(1000 * 30)
+ operations(NAME, loadAppReport(launchReportFile), desired, expected)
//stop
- freeze(0, CLUSTER,
+ freeze(0, NAME,
[
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "final-shutdown"
])
assertInYarnState(appId, YarnApplicationState.FINISHED)
- destroy(0, CLUSTER)
+ destroy(0, NAME)
//cluster now missing
- exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ exists(EXIT_UNKNOWN_INSTANCE, NAME)
}
protected int buildExpectedCount(int desired) {
@@ -122,9 +133,17 @@ public class AASleepIT extends AgentCommandTestBase
SerializedApplicationReport appReport,
int desired,
int expected ) {
- // sleep for some manual test
- describe("You may quickly perform manual tests against the application instance $CLUSTER")
- sleep(1000 * 30)
+
+
+ // now here await for the cluster size to grow: if it does, there's a problem
+ ClusterDescription cd
+ // spin for a while and fail if the number ever goes above it.
+ 5.times {
+ cd = assertContainersLive(NAME, SLEEP_LONG, expected)
+ sleep(1000 * 10)
+ }
+
+ // here cluster is still 1 below expected
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1977be/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 70f31d5..d1f8f20 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -43,7 +43,6 @@ import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.server.appmaster.rpc.RpcBinder
import org.junit.After
import org.junit.Before
-import org.junit.Ignore
import org.junit.Test
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_HEALTHCHECK