You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/10 18:51:51 UTC
[1/3] incubator-slider git commit: SLIDER-622 moved exec operations
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-622-windows 1ea195d2e -> 9cc24d008
SLIDER-622 moved exec operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/290a5e89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/290a5e89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/290a5e89
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 290a5e890d245304b00af99006709a0584241846
Parents: 1ea195d
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 10 17:50:12 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 10 17:50:12 2014 +0000
----------------------------------------------------------------------
.../apache/slider/test/SliderTestUtils.groovy | 41 ++++++++++++++++++++
1 file changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/290a5e89/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 634b6b5..8d6c036 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.service.ServiceStateException
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -46,6 +47,7 @@ import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.server.services.workflow.ForkedProcessService
import org.junit.Assert
import org.junit.Assume
@@ -520,6 +522,45 @@ class SliderTestUtils extends Assert {
int actual = instances != null ? instances.size() : 0
return actual
}
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
+ * @param commands
+ * @return the process
+ */
+ public static ForkedProcessService exec(int status, List<String> commands) {
+ ForkedProcessService process = exec(commands)
+ assert status == process.exitCode
+ return process
+ }
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+ public static ForkedProcessService exec(List<String> commands) {
+ ForkedProcessService process;
+ process = new ForkedProcessService(
+ commands[0],
+ [:],
+ commands);
+ process.init(new Configuration());
+ process.start();
+ process.waitForServiceToStop(10000);
+ process
+ }
+
+ public static boolean doesWindowsAppExist(List<String> commands) {
+ try {
+ exec(0, commands)
+ return true;
+ } catch (ServiceStateException e) {
+ if (!(e.cause instanceof FileNotFoundException)) {
+ throw e;
+ }
+ return false;
+ }
+ }
/**
* Execute a closure, assert it fails with a given exit code and text
[2/3] incubator-slider git commit: SLIDER-622 fixed erroneous negation
Posted by st...@apache.org.
SLIDER-622 fixed erroneous negation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/09ef9240
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/09ef9240
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/09ef9240
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 09ef9240d758a8d9edc9506b4a6d98935b648dcf
Parents: 290a5e8
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 10 17:50:28 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 10 17:50:28 2014 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/common/tools/SliderUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09ef9240/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index fa3f0bd..f4cea00 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1910,7 +1910,7 @@ public final class SliderUtils {
String errorText = null;
process.start();
try {
- if (process.waitForServiceToStop(timeoutMillis)) {
+ if (!process.waitForServiceToStop(timeoutMillis)) {
throw new TimeoutException(
"Process did not stop in " + timeoutMillis + "mS");
}
[3/3] incubator-slider git commit: SLIDER-622 kill operation tested
for at class load time;
used to skip kill() operations.Tests that use the kill explicilty must then be
made to skip if not kill_supported
Posted by st...@apache.org.
SLIDER-622 kill operation tested for at class load time; used to skip kill() operations.Tests that use the kill explicilty must then be made to skip if not kill_supported
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9cc24d00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9cc24d00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9cc24d00
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 9cc24d0080a1cf07ebd5970ed1d9b4e097660538
Parents: 09ef924
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 10 17:51:39 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 10 17:51:39 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAMKill.groovy | 4 +-
.../common/tools/TestWindowsSupport.groovy | 48 +++++-----------
.../apache/slider/test/SliderTestUtils.groovy | 38 -------------
.../slider/test/YarnMiniClusterTestBase.groovy | 60 +++++++++++++++++++-
.../providers/accumulo/AccumuloTestBase.groovy | 2 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
6 files changed, 80 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
index bad2715..d6b3929 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -27,6 +27,7 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Assume
import org.junit.Test
/**
@@ -38,9 +39,9 @@ import org.junit.Test
class TestStandaloneAMKill extends AgentMiniClusterTestBase {
-
@Test
public void testKillStandaloneAM() throws Throwable {
+ Assume.assumeTrue(kill_supported)
String clustername = createMiniCluster("", configuration, 1, true)
describe "kill a Standalone AM and verify that it shuts down"
@@ -54,6 +55,7 @@ class TestStandaloneAMKill extends AgentMiniClusterTestBase {
SliderClient sliderClient = launcher.service
addToTeardown(sliderClient);
ApplicationReport report = waitForClusterLive(sliderClient)
+ assert report.yarnApplicationState == YarnApplicationState.RUNNING
describe("listing Java processes")
lsJavaProcesses();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 25b7c57..9b383f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -28,15 +28,15 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.service.ServiceStateException
import org.apache.hadoop.util.Shell
import org.apache.slider.providers.agent.AgentUtils
-import org.apache.slider.server.services.workflow.ForkedProcessService
import org.apache.slider.test.SliderTestBase
+import org.apache.slider.test.YarnMiniClusterTestBase
import org.junit.Test
import java.util.regex.Pattern
@CompileStatic
@Slf4j
-class TestWindowsSupport extends SliderTestBase {
+class TestWindowsSupport extends YarnMiniClusterTestBase {
private static final Pattern hasDriveLetterSpecifier =
Pattern.compile("^/?[a-zA-Z]:");
@@ -121,18 +121,28 @@ class TestWindowsSupport extends SliderTestBase {
@Test
public void testExecNonexistentBinary() throws Throwable {
assume(Shell.WINDOWS, "not windows")
+ def commands = ["undefined-application", "--version"]
try {
- exec(2, ["undefined-application", "--version"])
+ exec(0, commands)
+ fail("expected an exception")
} catch (ServiceStateException e) {
if (!(e.cause instanceof FileNotFoundException)) {
throw e;
}
}
}
+ @Test
+ public void testExecNonexistentBinary2() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ assert !doesWindowsAppExist(["undefined-application", "--version"])
+ }
@Test
public void testEmitKillCommand() throws Throwable {
- killJavaProcesses("regionserver", 9)
+
+ def result = killJavaProcesses("regionserver", 9)
+ // we know the exit code if there is no supported kill operation
+ assert kill_supported || result == -1
}
@Test
@@ -159,34 +169,4 @@ class TestWindowsSupport extends SliderTestBase {
exec(0, [winUtilsPath, "systeminfo"])
}
-
- /**
- * Exec a set of commands, wait a few seconds for it to finish.
- * @param status code
- * @param commands
- * @return the process
- */
- public ForkedProcessService exec(int status, List<String> commands) {
- ForkedProcessService process = exec(commands)
- assert status == process.exitCode
- return process
- }
-
- /**
- * Exec a set of commands, wait a few seconds for it to finish.
- * @param commands
- * @return
- */
-
- public ForkedProcessService exec(List<String> commands) {
- ForkedProcessService process;
- process = new ForkedProcessService(
- methodName.methodName,
- [:],
- commands);
- process.init(new Configuration());
- process.start();
- process.waitForServiceToStop(10000);
- process
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 8d6c036..ad18c72 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -925,44 +925,6 @@ class SliderTestUtils extends Assert {
}
/**
- * Kill any java process with the given grep pattern
- * @param grepString string to grep for
- */
- public int killJavaProcesses(String grepString, int signal) {
-
- def commandString
- if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
- log.info("Command command = $killCommand")
-
- commandString = ["bash", "-c", killCommand]
- } else {
- /*
- "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
- */
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
- commandString = ["CMD", "/C", killCommand]
- }
- Process command = commandString.execute()
- def exitCode = command.waitFor()
-
- log.info(command.in.text)
- log.error(command.err.text)
- return exitCode
- }
-
- /**
- * Kill all processes which match one of the list of grepstrings
- * @param greps
- * @param signal
- */
- public void killJavaProcesses(List<String> greps, int signal) {
- for (String grep : greps) {
- killJavaProcesses(grep, signal)
- }
- }
-
- /**
* Convert a file to a URI suitable for use in an argument
* @param file file
* @return a URI string valid on all platforms
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index eb7cfa2..d5bad50 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.hdfs.MiniDFSCluster
import org.apache.hadoop.service.ServiceOperations
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -84,6 +85,9 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
public static final YarnConfiguration SLIDER_CONFIG = SliderUtils.createConfiguration();
+
+ public static boolean kill_supported;
+
static {
SLIDER_CONFIG.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 1)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100)
@@ -92,7 +96,6 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
SLIDER_CONFIG.setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
true)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1)
-
}
@@ -195,6 +198,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
protected void addToTeardown(SliderClient client) {
clustersToTeardown << client;
}
+
protected void addToTeardown(ServiceLauncher<SliderClient> launcher) {
SliderClient sliderClient = launcher?.service
if (sliderClient) {
@@ -202,6 +206,60 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
}
+ /**
+ * Work out if kill is supported
+ */
+ @BeforeClass
+ public static void checkKillSupport() {
+ if (!Shell.WINDOWS) {
+ kill_supported = true;
+ } else {
+ kill_supported = doesWindowsAppExist(["xargs", "--version"])
+ }
+ }
+
+ /**
+ * Kill any java process with the given grep pattern
+ * @param grepString string to grep for
+ */
+ public int killJavaProcesses(String grepString, int signal) {
+
+ def commandString
+ if (!Shell.WINDOWS) {
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
+ log.info("Command command = $killCommand")
+
+ commandString = ["bash", "-c", killCommand]
+ } else {
+ // windows
+ if (!kill_supported) {
+ return -1;
+ }
+
+ /*
+ "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
+ */
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
+ commandString = ["CMD", "/C", killCommand]
+ }
+ Process command = commandString.execute()
+ def exitCode = command.waitFor()
+
+ log.info(command.in.text)
+ log.error(command.err.text)
+ return exitCode
+ }
+
+ /**
+ * Kill all processes which match one of the list of grepstrings
+ * @param greps
+ * @param signal
+ */
+ public void killJavaProcesses(List<String> greps, int signal) {
+ for (String grep : greps) {
+ killJavaProcesses(grep, signal)
+ }
+ }
protected YarnConfiguration getConfiguration() {
return SLIDER_CONFIG;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
index 3c5606b..2a87cf0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -71,7 +71,7 @@ public abstract class AccumuloTestBase extends YarnZKMiniClusterTestBase {
@Override
void teardown() {
super.teardown();
- if (teardownKillall) {
+ if (teardownKillall && kill_supported) {
try {
killAllAccumuloProcesses();
} catch (AssumptionViolatedException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cc24d00/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
index 3d0c03f..fcb73e0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -87,7 +87,9 @@ class TestAccFreezeThaw extends AccumuloTestBase {
log.debug("expected exception", expected)
}
//force kill any accumulo processes
- killAllAccumuloProcesses()
+ if (kill_supported) {
+ killAllAccumuloProcesses()
+ }
sleepForAccumuloClusterLive();