You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/11 15:12:02 UTC
[1/6] incubator-slider git commit: SLIDER-622 warn if the directory
is still present after the delete (shutdown race condition problem on NTFS)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-622-windows 8a3258ee7 -> 3dd6ac495
SLIDER-622 warn if the directory is still present after the delete (shutdown race condition problem on NTFS)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2768a16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2768a16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2768a16
Branch: refs/heads/feature/SLIDER-622-windows
Commit: b2768a165f09cb181446f812f2a310eab5e0f2eb
Parents: 8a3258e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 12:58:24 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 12:58:24 2014 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2768a16/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index a676fcc..164e2fe 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsAction;
@@ -536,24 +537,23 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
// delete the directory;
- boolean exists = sliderFileSystem.getFileSystem().exists(clusterDirectory);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ boolean exists = fs.exists(clusterDirectory);
if (exists) {
log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
boolean deleted =
- sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
+ fs.delete(clusterDirectory, true);
if (!deleted) {
log.warn("Filesystem returned false from delete() operation");
}
-
- if(!deleteZookeeperNode(clustername)) {
- log.warn("Unable to perform node cleanup in Zookeeper.");
+ if (fs.exists(clusterDirectory)) {
+ log.warn("Failed to delete {}", clusterDirectory);
}
} else {
log.debug("Application Instance {} already destroyed", clustername);
}
-
// rm the registry entry —do not let this block the destroy operations
String registryPath = SliderRegistryUtils.registryPathForInstance(
clustername);
[4/6] incubator-slider git commit: SLIDER-637 rm unneeded test in
TestZKIntegration
Posted by st...@apache.org.
SLIDER-637 rm unneeded test in TestZKIntegration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2bb2323c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2bb2323c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2bb2323c
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 2bb2323ccd0921da56bfdc502ece5d5b96ee8237
Parents: f9d7d45
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:30 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:50:30 2014 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/TestZKIntegration.groovy | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2bb2323c/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 0aa1b91..431c49f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -59,16 +59,6 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
zki = null;
}
- @Test
- public void testIntegrationCreate() throws Throwable {
- assertHasZKCluster()
- initZKI()
- String userPath = ZKIntegration.mkSliderUserPath(USER)
- Stat stat = zki.stat(userPath)
- assert stat != null
- log.info("User path $userPath has stat $stat")
- }
-
public ZKIntegration initZKI() {
zki = createZKIntegrationInstance(
getZKBinding(), methodName.methodName, true, false, 5000)
[3/6] incubator-slider git commit: Merge branch 'develop' into
feature/SLIDER-622-windows
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-622-windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f9d7d450
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f9d7d450
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f9d7d450
Branch: refs/heads/feature/SLIDER-622-windows
Commit: f9d7d4503491fbdcbb462d828996157d3b5d462d
Parents: b2768a1 56bc48d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:37:54 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:37:54 2014 +0000
----------------------------------------------------------------------
.../live/TestAccCorrectInstanceName.groovy | 17 +++--------------
.../accumulo/live/TestAccFreezeThaw.groovy | 6 +-----
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 8 ++------
.../accumulo/live/TestAccLiveLocalArchive.groovy | 8 ++------
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccumuloAMWebApp.groovy | 10 +++-------
.../accumulo/live/TestInvalidMonitorAddress.groovy | 10 +++++-----
.../hbase/minicluster/live/Test2Master2RS.groovy | 6 +++---
.../hbase/minicluster/live/TestHBaseMaster.groovy | 11 ++++-------
.../minicluster/live/TestHBaseMasterOnHDFS.groovy | 2 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 5 +----
.../live/TestLiveTwoNodeRegionService.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 6 +++---
14 files changed, 35 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
[6/6] incubator-slider git commit: SLIDER-622 add spin-checks for
cluster dir deletion...deals with NTFS problems
Posted by st...@apache.org.
SLIDER-622 add spin-checks for cluster dir deletion...deals with NTFS problems
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd6ac49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd6ac49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd6ac49
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 3dd6ac495ea2b87ced7069f19ce04cf8d52559f2
Parents: 749484c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:11:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 14:11:44 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAMDestroy.groovy | 5 ++++
.../slider/test/YarnMiniClusterTestBase.groovy | 29 ++++++++++----------
2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd6ac49/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 4a94eb3..d25396d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -121,6 +121,11 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
assert 0 == exitCode
describe "post destroy checks"
+ if (fs.exists(instanceDir)) {
+ log.warn("Destroy operation did not delete $instanceDir")
+ rigorousDelete(sliderFileSystem, instanceDir, 60000)
+ }
+
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
// look up app report and verify exit code is good
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd6ac49/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 2805304..a816a0d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -510,11 +510,13 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
def config = miniCluster.config
if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), config)
- Path clusterDir = new SliderFileSystem(dfs, config).buildClusterDirPath(clustername)
- log.info("deleting customer data at $clusterDir")
+
+ def sliderFileSystem = new SliderFileSystem(dfs, config)
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername)
+ log.info("deleting instance data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- rigorousDelete(dfs, clusterDir, 60000)
+ rigorousDelete(sliderFileSystem, clusterDir, 60000)
}
@@ -565,26 +567,23 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
* Delete with some pauses and backoff; designed to handle slow delete
* operation in windows
* @param dfs
- * @param clusterDir
+ * @param path
*/
public void rigorousDelete(
- HadoopFS dfs, Path clusterDir, long timeout) {
+ SliderFileSystem sliderFileSystem,
+ Path path, long timeout) {
Duration duration = new Duration(timeout)
duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
boolean deleted = false;
- while (!deleted) {
- dfs.delete(clusterDir, true)
- deleted = !dfs.exists(clusterDir)
- if (!deleted ) {
- if (duration.limitExceeded) {
- assert !dfs.exists(
- clusterDir), "delete operation failed —directory in use?"
-
- }
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
sleep(1000)
}
-
}
+ sliderFileSystem.verifyDirectoryNonexistent(path)
}
/**
[2/6] incubator-slider git commit: SLIDER-637 purge ZKInstance uses
in the accumulo and hbase providers
Posted by st...@apache.org.
SLIDER-637 purge ZKInstance uses in the accumulo and hbase providers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56bc48dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56bc48dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56bc48dc
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 56bc48dc4fd1c34bc4f5115242081ed8adb57277
Parents: d608dba
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:33:39 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:33:39 2014 +0000
----------------------------------------------------------------------
.../live/TestAccCorrectInstanceName.groovy | 17 +++--------------
.../accumulo/live/TestAccFreezeThaw.groovy | 6 +-----
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 8 ++------
.../accumulo/live/TestAccLiveLocalArchive.groovy | 8 ++------
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccumuloAMWebApp.groovy | 10 +++-------
.../accumulo/live/TestInvalidMonitorAddress.groovy | 10 +++++-----
.../hbase/minicluster/live/Test2Master2RS.groovy | 6 +++---
.../hbase/minicluster/live/TestHBaseMaster.groovy | 11 ++++-------
.../minicluster/live/TestHBaseMasterOnHDFS.groovy | 2 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 5 +----
.../live/TestLiveTwoNodeRegionService.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 6 +++---
14 files changed, 35 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
index 2333fdf..1f1b726 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
@@ -20,17 +20,13 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.accumulo.core.client.Connector
-import org.apache.accumulo.core.client.ZooKeeperInstance
-import org.apache.accumulo.core.client.security.tokens.PasswordToken
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -47,8 +43,6 @@ class TestAccCorrectInstanceName extends AccumuloTestBase {
describe(" Create an accumulo cluster");
//make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
@@ -79,11 +73,6 @@ class TestAccCorrectInstanceName extends AccumuloTestBase {
//verify that all is still there
waitForRoleCount(sliderClient, roles, 0)
- // Making the connector validates that the instance name is correct, we have the right zk,
- // and the proper username and password were created in accumulo
- ZooKeeperInstance instance = new ZooKeeperInstance(System.getProperty("user.name") + "-" + clustername, zki.getConnectionString());
- Connector c = instance.getConnector("root", new PasswordToken(password));
-
log.info("Finishing")
status = sliderClient.getClusterDescription(clustername)
log.info(prettyPrint(status.toJsonString()))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
index fcb73e0..f55b73e 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -21,12 +21,11 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -42,9 +41,6 @@ class TestAccFreezeThaw extends AccumuloTestBase {
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
index 8d5890c..ecb9062 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@ class TestAccLiveHDFSArchive extends AccumuloTestBase {
describe(" Create an accumulo cluster from an archive");
enableTestRunAgainstUploadedArchive();
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
index df0bbd9..997fa19 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -46,9 +45,6 @@ class TestAccLiveLocalArchive extends AccumuloTestBase {
//image mode
switchToImageDeploy = true
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
index d94eb36..586c4c0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
@@ -21,12 +21,11 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -41,9 +40,6 @@ class TestAccM1T1GC1Mon1 extends AccumuloTestBase {
String clustername = createMiniCluster( "", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
index 2ed50f1..804d29d 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@ class TestAccM2T2GC1Mon1 extends AccumuloTestBase {
"", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): master,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
index 4596b12..7afcd0f 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
@@ -22,14 +22,13 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
+import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.junit.Test
@CompileStatic
@@ -45,9 +44,6 @@ class TestAccumuloAMWebApp extends AccumuloTestBase {
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
index dca0c6b..a07ddc9 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
@@ -20,14 +20,14 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.core.main.ServiceLaunchException
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
index 47f5e1c..10a502b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
@@ -22,11 +22,11 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.ClusterStatus
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.hbase.HBaseTestUtils
-import org.apache.slider.common.tools.Duration
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 79193c3..db102cb 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -23,17 +23,16 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.core.zk.ZKIntegration
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
/**
@@ -47,8 +46,6 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
@Test
public void testHBaseMaster() throws Throwable {
String clustername = createMiniCluster("", configuration, 1, true)
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster with 1 region server
int regionServerCount = 1
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
index 7f51f95..5e8adca 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
@@ -22,8 +22,8 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
index 957d167..1392535 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
@@ -22,10 +22,9 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
@@ -42,8 +41,6 @@ class TestLiveRegionServiceOnHDFS extends HBaseMiniClusterTestBase {
"", configuration, 1, 1, 1, true, true)
describe(" Create a single region service cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
SliderClient sliderClient = launcher.service
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
index 3561d2f..f10e296 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
@@ -23,11 +23,11 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56bc48dc/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index b7212d9..029f424 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -21,15 +21,15 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.api.RegistryOperations
-import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@CompileStatic
[5/6] incubator-slider git commit: SLIDER-622 spin for deletion
Posted by st...@apache.org.
SLIDER-622 spin for deletion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/749484c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/749484c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/749484c2
Branch: refs/heads/feature/SLIDER-622-windows
Commit: 749484c2df3e170f30bba71afa9c5a242238af3b
Parents: 2bb2323
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:59 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:50:59 2014 +0000
----------------------------------------------------------------------
.../slider/test/YarnMiniClusterTestBase.groovy | 32 +++++++++++++++++---
1 file changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/749484c2/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index dcc55f6..2805304 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -18,7 +18,6 @@
package org.apache.slider.test
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
@@ -515,10 +514,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
log.info("deleting customer data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- dfs.delete(clusterDir, true)
- sleep(1000)
- dfs.delete(clusterDir, true)
- assert !dfs.exists(clusterDir), "delete operation failed —application in use?"
+ rigorousDelete(dfs, clusterDir, 60000)
}
@@ -566,6 +562,32 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
/**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dfs
+ * @param clusterDir
+ */
+ public void rigorousDelete(
+ HadoopFS dfs, Path clusterDir, long timeout) {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ boolean deleted = false;
+ while (!deleted) {
+ dfs.delete(clusterDir, true)
+ deleted = !dfs.exists(clusterDir)
+ if (!deleted ) {
+ if (duration.limitExceeded) {
+ assert !dfs.exists(
+ clusterDir), "delete operation failed —directory in use?"
+
+ }
+ sleep(1000)
+ }
+
+ }
+ }
+
+ /**
* Add arguments to launch Slider with.
*
* Extra arguments are added after standard arguments and before roles.