You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/11 16:05:04 UTC
[01/17] incubator-slider git commit: SLIDER-622: fix ZK Integration
tests to be more independent and better @ cleaning up
Repository: incubator-slider
Updated Branches:
refs/heads/develop 56bc48dc4 -> 36e5d4351
refs/heads/releases/slider-0.60 e04c51b0e -> 56e16c391
SLIDER-622: fix ZK Integration tests to be more independent and better @ cleaning up
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8a3258ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8a3258ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8a3258ee
Branch: refs/heads/develop
Commit: 8a3258ee73e723d4b583493c6a993e3fe762998c
Parents: d608dba
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 11:40:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 11:40:44 2014 +0000
----------------------------------------------------------------------
.../apache/slider/core/zk/ZKIntegration.java | 18 ++++++++-
.../common/tools/TestZKIntegration.groovy | 41 +++++++++++++-------
2 files changed, 44 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8a3258ee/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 0d96559..0c28229 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -29,13 +29,14 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-public class ZKIntegration implements Watcher {
+public class ZKIntegration implements Watcher, Closeable {
/**
* Base path for services
@@ -115,7 +116,20 @@ public class ZKIntegration implements Watcher {
createClusterPath,
watchEventHandler);
}
-
+
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (zookeeper != null) {
+ try {
+ zookeeper.close();
+ } catch (InterruptedException ignored) {
+
+ }
+ zookeeper = null;
+ }
+ }
+
public String getConnectionString() {
return zkConnection;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8a3258ee/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 460dafb..0aa1b91 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -21,6 +21,8 @@ package org.apache.slider.common.tools
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.test.KeysForTests
@@ -28,6 +30,7 @@ import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.apache.zookeeper.CreateMode
import org.apache.zookeeper.ZooDefs
import org.apache.zookeeper.data.Stat
+import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -38,32 +41,46 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
// as the static compiler doesn't resolve consistently
public static final String USER = KeysForTests.USERNAME
+ private ZKIntegration zki
@Before
void createCluster() {
Configuration conf = configuration
- createMicroZKCluster(methodName.methodName, conf)
+ def name = methodName.methodName
+ File zkdir = new File("target/zk/${name}")
+ FileUtil.fullyDelete(zkdir);
+ conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir.absolutePath)
+ createMicroZKCluster("-"+ name, conf)
+ }
+
+ @After
+ void closeZKI() {
+ zki?.close()
+ zki = null;
}
@Test
public void testIntegrationCreate() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), "cluster1", true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
Stat stat = zki.stat(userPath)
assert stat != null
log.info("User path $userPath has stat $stat")
}
+ public ZKIntegration initZKI() {
+ zki = createZKIntegrationInstance(
+ getZKBinding(), methodName.methodName, true, false, 5000)
+ return zki
+ }
+
@Test
public void testListUserClustersWithoutAnyClusters() throws Throwable {
assertHasZKCluster()
-
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), "", true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
- List<String> clusters = zki.clusters
+ List<String> clusters = this.zki.clusters
assert clusters.empty
}
@@ -71,8 +88,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
public void testListUserClustersWithOneCluster() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), methodName.methodName, true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
@@ -85,8 +101,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), methodName.methodName, true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
@@ -103,7 +118,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
MockSliderClient client = new MockSliderClient()
String path = client.createZookeeperNode("cl1", true)
- ZKIntegration zki = client.lastZKIntegration
+ zki = client.lastZKIntegration
String zkPath = ZKIntegration.mkClusterPath(USER, "cl1")
assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
@@ -140,7 +155,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000)
+ zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, false, 5000)
return zki;
}
[02/17] incubator-slider git commit: SLIDER-622 warn if the directory
is still present after the delete (shutdown race condition problem on NTFS)
Posted by st...@apache.org.
SLIDER-622 warn if the directory is still present after the delete (shutdown race condition problem on NTFS)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2768a16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2768a16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2768a16
Branch: refs/heads/develop
Commit: b2768a165f09cb181446f812f2a310eab5e0f2eb
Parents: 8a3258e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 12:58:24 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 12:58:24 2014 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2768a16/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index a676fcc..164e2fe 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsAction;
@@ -536,24 +537,23 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
// delete the directory;
- boolean exists = sliderFileSystem.getFileSystem().exists(clusterDirectory);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ boolean exists = fs.exists(clusterDirectory);
if (exists) {
log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
boolean deleted =
- sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
+ fs.delete(clusterDirectory, true);
if (!deleted) {
log.warn("Filesystem returned false from delete() operation");
}
-
- if(!deleteZookeeperNode(clustername)) {
- log.warn("Unable to perform node cleanup in Zookeeper.");
+ if (fs.exists(clusterDirectory)) {
+ log.warn("Failed to delete {}", clusterDirectory);
}
} else {
log.debug("Application Instance {} already destroyed", clustername);
}
-
// rm the registry entry —do not let this block the destroy operations
String registryPath = SliderRegistryUtils.registryPathForInstance(
clustername);
[09/17] incubator-slider git commit: Merge branch
'feature/SLIDER-622-windows' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-622-windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/36e5d435
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/36e5d435
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/36e5d435
Branch: refs/heads/develop
Commit: 36e5d435196fb6068bf00acb01fd52d1dd7c0fa5
Parents: 56bc48d eba57bb
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 15:02:23 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:02:23 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 12 ++--
.../slider/core/build/InstanceBuilder.java | 4 +-
.../apache/slider/core/zk/ZKIntegration.java | 18 +++++-
.../standalone/TestStandaloneAMDestroy.groovy | 5 ++
.../common/tools/TestZKIntegration.groovy | 45 ++++++++------
.../providers/agent/TestBuildBasicAgent.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 65 +++++++++++++++++---
7 files changed, 114 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
[05/17] incubator-slider git commit: SLIDER-622 spin for deletion
Posted by st...@apache.org.
SLIDER-622 spin for deletion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/749484c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/749484c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/749484c2
Branch: refs/heads/develop
Commit: 749484c2df3e170f30bba71afa9c5a242238af3b
Parents: 2bb2323
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:59 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:50:59 2014 +0000
----------------------------------------------------------------------
.../slider/test/YarnMiniClusterTestBase.groovy | 32 +++++++++++++++++---
1 file changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/749484c2/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index dcc55f6..2805304 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -18,7 +18,6 @@
package org.apache.slider.test
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
@@ -515,10 +514,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
log.info("deleting customer data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- dfs.delete(clusterDir, true)
- sleep(1000)
- dfs.delete(clusterDir, true)
- assert !dfs.exists(clusterDir), "delete operation failed —application in use?"
+ rigorousDelete(dfs, clusterDir, 60000)
}
@@ -566,6 +562,32 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
/**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dfs
+ * @param clusterDir
+ */
+ public void rigorousDelete(
+ HadoopFS dfs, Path clusterDir, long timeout) {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ boolean deleted = false;
+ while (!deleted) {
+ dfs.delete(clusterDir, true)
+ deleted = !dfs.exists(clusterDir)
+ if (!deleted ) {
+ if (duration.limitExceeded) {
+ assert !dfs.exists(
+ clusterDir), "delete operation failed —directory in use?"
+
+ }
+ sleep(1000)
+ }
+
+ }
+ }
+
+ /**
* Add arguments to launch Slider with.
*
* Extra arguments are added after standard arguments and before roles.
[10/17] incubator-slider git commit: SLIDER-622: fix ZK Integration
tests to be more independent and better @ cleaning up
Posted by st...@apache.org.
SLIDER-622: fix ZK Integration tests to be more independent and better @ cleaning up
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b88c7759
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b88c7759
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b88c7759
Branch: refs/heads/releases/slider-0.60
Commit: b88c775949bd2fd9d0605b2d8ea4c86724cbc42b
Parents: e04c51b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 11:40:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:19 2014 +0000
----------------------------------------------------------------------
.../apache/slider/core/zk/ZKIntegration.java | 18 ++++++++-
.../common/tools/TestZKIntegration.groovy | 41 +++++++++++++-------
2 files changed, 44 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b88c7759/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 0d96559..0c28229 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -29,13 +29,14 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-public class ZKIntegration implements Watcher {
+public class ZKIntegration implements Watcher, Closeable {
/**
* Base path for services
@@ -115,7 +116,20 @@ public class ZKIntegration implements Watcher {
createClusterPath,
watchEventHandler);
}
-
+
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (zookeeper != null) {
+ try {
+ zookeeper.close();
+ } catch (InterruptedException ignored) {
+
+ }
+ zookeeper = null;
+ }
+ }
+
public String getConnectionString() {
return zkConnection;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b88c7759/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 460dafb..0aa1b91 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -21,6 +21,8 @@ package org.apache.slider.common.tools
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.test.KeysForTests
@@ -28,6 +30,7 @@ import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.apache.zookeeper.CreateMode
import org.apache.zookeeper.ZooDefs
import org.apache.zookeeper.data.Stat
+import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -38,32 +41,46 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
// as the static compiler doesn't resolve consistently
public static final String USER = KeysForTests.USERNAME
+ private ZKIntegration zki
@Before
void createCluster() {
Configuration conf = configuration
- createMicroZKCluster(methodName.methodName, conf)
+ def name = methodName.methodName
+ File zkdir = new File("target/zk/${name}")
+ FileUtil.fullyDelete(zkdir);
+ conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir.absolutePath)
+ createMicroZKCluster("-"+ name, conf)
+ }
+
+ @After
+ void closeZKI() {
+ zki?.close()
+ zki = null;
}
@Test
public void testIntegrationCreate() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), "cluster1", true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
Stat stat = zki.stat(userPath)
assert stat != null
log.info("User path $userPath has stat $stat")
}
+ public ZKIntegration initZKI() {
+ zki = createZKIntegrationInstance(
+ getZKBinding(), methodName.methodName, true, false, 5000)
+ return zki
+ }
+
@Test
public void testListUserClustersWithoutAnyClusters() throws Throwable {
assertHasZKCluster()
-
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), "", true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
- List<String> clusters = zki.clusters
+ List<String> clusters = this.zki.clusters
assert clusters.empty
}
@@ -71,8 +88,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
public void testListUserClustersWithOneCluster() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), methodName.methodName, true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
@@ -85,8 +101,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
- ZKIntegration zki = createZKIntegrationInstance(
- getZKBinding(), methodName.methodName, true, false, 5000)
+ initZKI()
String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
@@ -103,7 +118,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
MockSliderClient client = new MockSliderClient()
String path = client.createZookeeperNode("cl1", true)
- ZKIntegration zki = client.lastZKIntegration
+ zki = client.lastZKIntegration
String zkPath = ZKIntegration.mkClusterPath(USER, "cl1")
assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
@@ -140,7 +155,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000)
+ zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, false, 5000)
return zki;
}
[08/17] incubator-slider git commit: SLIDER-622 ensuring unique names
for instances in a test
Posted by st...@apache.org.
SLIDER-622 ensuring unique names for instances in a test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eba57bb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eba57bb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eba57bb5
Branch: refs/heads/develop
Commit: eba57bb508fc26ee96affcc79a5374af900d05d8
Parents: 89073b1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:50:16 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 14:50:16 2014 +0000
----------------------------------------------------------------------
.../main/java/org/apache/slider/core/build/InstanceBuilder.java | 4 +++-
.../org/apache/slider/providers/agent/TestBuildBasicAgent.groovy | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba57bb5/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 669a5ad..cee2663 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -253,7 +253,9 @@ public class InstanceBuilder {
IOException,
SliderException,
LockAcquireFailedException {
- if (!overwrite) coreFS.createClusterDirectories(instancePaths);
+ if (!overwrite) {
+ coreFS.createClusterDirectories(instancePaths);
+ }
ConfPersister persister =
new ConfPersister(coreFS, getInstanceDir());
ConfDirSnapshotAction action = null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba57bb5/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 3f7a432..264d260 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -171,7 +171,7 @@ class TestBuildBasicAgent extends AgentTestBase {
}
try {
- launcher = buildAgentCluster(clustername,
+ launcher = buildAgentCluster(clustername + "-10",
[
(ROLE_NODE): 4,
],
[07/17] incubator-slider git commit: SLIDER-622 bypass Haddop APIs
and go to commons IO directory delete if the path is file://
Posted by st...@apache.org.
SLIDER-622 bypass Haddop APIs and go to commons IO directory delete if the path is file://
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/89073b13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/89073b13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/89073b13
Branch: refs/heads/develop
Commit: 89073b13444728eded65d93373058bac227585ec
Parents: 3dd6ac4
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:37:28 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 14:37:28 2014 +0000
----------------------------------------------------------------------
.../slider/test/YarnMiniClusterTestBase.groovy | 38 +++++++++++++++++---
1 file changed, 34 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/89073b13/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index a816a0d..856ad11 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.test
import groovy.util.logging.Slf4j
+import org.apache.commons.io.FileUtils
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.conf.Configuration
@@ -572,18 +573,47 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
public void rigorousDelete(
SliderFileSystem sliderFileSystem,
Path path, long timeout) {
+
+ if (path.toUri().scheme == "file") {
+ File dir = new File(path.toUri().getPath());
+ rigorousDelete(dir, timeout)
+ } else {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
+ boolean deleted = false;
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
+ sleep(1000)
+ }
+ }
+ }
+ sliderFileSystem.verifyDirectoryNonexistent(path)
+ }
+
+ /**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dir dir to delete
+ * @param timeout timeout in millis
+ */
+ public void rigorousDelete(File dir, long timeout) {
Duration duration = new Duration(timeout)
duration.start()
- HadoopFS dfs = sliderFileSystem.fileSystem
boolean deleted = false;
while (!deleted && !duration.limitExceeded) {
- dfs.delete(path, true)
- deleted = !dfs.exists(path)
+ FileUtils.deleteQuietly(dir)
+ deleted = !dir.exists()
if (!deleted) {
sleep(1000)
}
}
- sliderFileSystem.verifyDirectoryNonexistent(path)
+ if (!deleted) {
+ // noisy delete raises an IOE
+ FileUtils.deleteDirectory(dir)
+ }
}
/**
[12/17] incubator-slider git commit: SLIDER-637 purge ZKInstance uses
in the accumulo and hbase providers
Posted by st...@apache.org.
SLIDER-637 purge ZKInstance uses in the accumulo and hbase providers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/62c87ec9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/62c87ec9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/62c87ec9
Branch: refs/heads/releases/slider-0.60
Commit: 62c87ec98dee5f0755dd54ca8f8918e461adc3b3
Parents: d556971
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:33:39 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:36 2014 +0000
----------------------------------------------------------------------
.../live/TestAccCorrectInstanceName.groovy | 17 +++--------------
.../accumulo/live/TestAccFreezeThaw.groovy | 6 +-----
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 8 ++------
.../accumulo/live/TestAccLiveLocalArchive.groovy | 8 ++------
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccumuloAMWebApp.groovy | 10 +++-------
.../accumulo/live/TestInvalidMonitorAddress.groovy | 10 +++++-----
.../hbase/minicluster/live/Test2Master2RS.groovy | 6 +++---
.../hbase/minicluster/live/TestHBaseMaster.groovy | 11 ++++-------
.../minicluster/live/TestHBaseMasterOnHDFS.groovy | 2 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 5 +----
.../live/TestLiveTwoNodeRegionService.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 6 +++---
14 files changed, 35 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
index 2333fdf..1f1b726 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
@@ -20,17 +20,13 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.accumulo.core.client.Connector
-import org.apache.accumulo.core.client.ZooKeeperInstance
-import org.apache.accumulo.core.client.security.tokens.PasswordToken
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -47,8 +43,6 @@ class TestAccCorrectInstanceName extends AccumuloTestBase {
describe(" Create an accumulo cluster");
//make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
@@ -79,11 +73,6 @@ class TestAccCorrectInstanceName extends AccumuloTestBase {
//verify that all is still there
waitForRoleCount(sliderClient, roles, 0)
- // Making the connector validates that the instance name is correct, we have the right zk,
- // and the proper username and password were created in accumulo
- ZooKeeperInstance instance = new ZooKeeperInstance(System.getProperty("user.name") + "-" + clustername, zki.getConnectionString());
- Connector c = instance.getConnector("root", new PasswordToken(password));
-
log.info("Finishing")
status = sliderClient.getClusterDescription(clustername)
log.info(prettyPrint(status.toJsonString()))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
index fcb73e0..f55b73e 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -21,12 +21,11 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -42,9 +41,6 @@ class TestAccFreezeThaw extends AccumuloTestBase {
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
index 8d5890c..ecb9062 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@ class TestAccLiveHDFSArchive extends AccumuloTestBase {
describe(" Create an accumulo cluster from an archive");
enableTestRunAgainstUploadedArchive();
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
index df0bbd9..997fa19 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -46,9 +45,6 @@ class TestAccLiveLocalArchive extends AccumuloTestBase {
//image mode
switchToImageDeploy = true
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
index d94eb36..586c4c0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
@@ -21,12 +21,11 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -41,9 +40,6 @@ class TestAccM1T1GC1Mon1 extends AccumuloTestBase {
String clustername = createMiniCluster( "", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
index 2ed50f1..804d29d 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
@@ -21,13 +21,12 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@ class TestAccM2T2GC1Mon1 extends AccumuloTestBase {
"", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): master,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
index 4596b12..7afcd0f 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
@@ -22,14 +22,13 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
+import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.junit.Test
@CompileStatic
@@ -45,9 +44,6 @@ class TestAccumuloAMWebApp extends AccumuloTestBase {
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
index dca0c6b..a07ddc9 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
@@ -20,14 +20,14 @@ package org.apache.slider.providers.accumulo.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.core.main.ServiceLaunchException
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
index 47f5e1c..10a502b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
@@ -22,11 +22,11 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.ClusterStatus
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.hbase.HBaseTestUtils
-import org.apache.slider.common.tools.Duration
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 79193c3..db102cb 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -23,17 +23,16 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.core.zk.ZKIntegration
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
/**
@@ -47,8 +46,6 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
@Test
public void testHBaseMaster() throws Throwable {
String clustername = createMiniCluster("", configuration, 1, true)
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster with 1 region server
int regionServerCount = 1
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
index 7f51f95..5e8adca 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
@@ -22,8 +22,8 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
index 957d167..1392535 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
@@ -22,10 +22,9 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
@@ -42,8 +41,6 @@ class TestLiveRegionServiceOnHDFS extends HBaseMiniClusterTestBase {
"", configuration, 1, 1, 1, true, true)
describe(" Create a single region service cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
SliderClient sliderClient = launcher.service
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
index 3561d2f..f10e296 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
@@ -23,11 +23,11 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62c87ec9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index b7212d9..029f424 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -21,15 +21,15 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.api.RegistryOperations
-import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@CompileStatic
[13/17] incubator-slider git commit: SLIDER-622 spin for deletion
Posted by st...@apache.org.
SLIDER-622 spin for deletion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/75963870
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/75963870
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/75963870
Branch: refs/heads/releases/slider-0.60
Commit: 7596387040446aa8bac157f09fc028fe4b699981
Parents: 632346e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:59 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:47 2014 +0000
----------------------------------------------------------------------
.../slider/test/YarnMiniClusterTestBase.groovy | 32 +++++++++++++++++---
1 file changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/75963870/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index dde8239..0850d25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -18,7 +18,6 @@
package org.apache.slider.test
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
@@ -493,10 +492,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
log.info("deleting customer data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- dfs.delete(clusterDir, true)
- sleep(1000)
- dfs.delete(clusterDir, true)
- assert !dfs.exists(clusterDir), "delete operation failed —application in use?"
+ rigorousDelete(dfs, clusterDir, 60000)
}
@@ -544,6 +540,32 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
/**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dfs
+ * @param clusterDir
+ */
+ public void rigorousDelete(
+ HadoopFS dfs, Path clusterDir, long timeout) {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ boolean deleted = false;
+ while (!deleted) {
+ dfs.delete(clusterDir, true)
+ deleted = !dfs.exists(clusterDir)
+ if (!deleted ) {
+ if (duration.limitExceeded) {
+ assert !dfs.exists(
+ clusterDir), "delete operation failed —directory in use?"
+
+ }
+ sleep(1000)
+ }
+
+ }
+ }
+
+ /**
* Add arguments to launch Slider with.
*
* Extra arguments are added after standard arguments and before roles.
[03/17] incubator-slider git commit: Merge branch 'develop' into
feature/SLIDER-622-windows
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-622-windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f9d7d450
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f9d7d450
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f9d7d450
Branch: refs/heads/develop
Commit: f9d7d4503491fbdcbb462d828996157d3b5d462d
Parents: b2768a1 56bc48d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:37:54 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:37:54 2014 +0000
----------------------------------------------------------------------
.../live/TestAccCorrectInstanceName.groovy | 17 +++--------------
.../accumulo/live/TestAccFreezeThaw.groovy | 6 +-----
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 8 ++------
.../accumulo/live/TestAccLiveLocalArchive.groovy | 8 ++------
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 8 ++------
.../accumulo/live/TestAccumuloAMWebApp.groovy | 10 +++-------
.../accumulo/live/TestInvalidMonitorAddress.groovy | 10 +++++-----
.../hbase/minicluster/live/Test2Master2RS.groovy | 6 +++---
.../hbase/minicluster/live/TestHBaseMaster.groovy | 11 ++++-------
.../minicluster/live/TestHBaseMasterOnHDFS.groovy | 2 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 5 +----
.../live/TestLiveTwoNodeRegionService.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 6 +++---
14 files changed, 35 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
[06/17] incubator-slider git commit: SLIDER-622 add spin-checks for
cluster dir deletion...deals with NTFS problems
Posted by st...@apache.org.
SLIDER-622 add spin-checks for cluster dir deletion...deals with NTFS problems
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd6ac49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd6ac49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd6ac49
Branch: refs/heads/develop
Commit: 3dd6ac495ea2b87ced7069f19ce04cf8d52559f2
Parents: 749484c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:11:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 14:11:44 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAMDestroy.groovy | 5 ++++
.../slider/test/YarnMiniClusterTestBase.groovy | 29 ++++++++++----------
2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd6ac49/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 4a94eb3..d25396d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -121,6 +121,11 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
assert 0 == exitCode
describe "post destroy checks"
+ if (fs.exists(instanceDir)) {
+ log.warn("Destroy operation did not delete $instanceDir")
+ rigorousDelete(sliderFileSystem, instanceDir, 60000)
+ }
+
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
// look up app report and verify exit code is good
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd6ac49/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 2805304..a816a0d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -510,11 +510,13 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
def config = miniCluster.config
if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), config)
- Path clusterDir = new SliderFileSystem(dfs, config).buildClusterDirPath(clustername)
- log.info("deleting customer data at $clusterDir")
+
+ def sliderFileSystem = new SliderFileSystem(dfs, config)
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername)
+ log.info("deleting instance data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- rigorousDelete(dfs, clusterDir, 60000)
+ rigorousDelete(sliderFileSystem, clusterDir, 60000)
}
@@ -565,26 +567,23 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
* Delete with some pauses and backoff; designed to handle slow delete
* operation in windows
* @param dfs
- * @param clusterDir
+ * @param path
*/
public void rigorousDelete(
- HadoopFS dfs, Path clusterDir, long timeout) {
+ SliderFileSystem sliderFileSystem,
+ Path path, long timeout) {
Duration duration = new Duration(timeout)
duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
boolean deleted = false;
- while (!deleted) {
- dfs.delete(clusterDir, true)
- deleted = !dfs.exists(clusterDir)
- if (!deleted ) {
- if (duration.limitExceeded) {
- assert !dfs.exists(
- clusterDir), "delete operation failed —directory in use?"
-
- }
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
sleep(1000)
}
-
}
+ sliderFileSystem.verifyDirectoryNonexistent(path)
}
/**
[14/17] incubator-slider git commit: SLIDER-637 rm unneeded test in
TestZKIntegration
Posted by st...@apache.org.
SLIDER-637 rm unneeded test in TestZKIntegration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/632346e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/632346e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/632346e5
Branch: refs/heads/releases/slider-0.60
Commit: 632346e540e1cac1ae6bac6b8ce1ed5bc5ba9772
Parents: 62c87ec
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:30 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:47 2014 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/TestZKIntegration.groovy | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/632346e5/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 0aa1b91..431c49f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -59,16 +59,6 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
zki = null;
}
- @Test
- public void testIntegrationCreate() throws Throwable {
- assertHasZKCluster()
- initZKI()
- String userPath = ZKIntegration.mkSliderUserPath(USER)
- Stat stat = zki.stat(userPath)
- assert stat != null
- log.info("User path $userPath has stat $stat")
- }
-
public ZKIntegration initZKI() {
zki = createZKIntegrationInstance(
getZKBinding(), methodName.methodName, true, false, 5000)
[16/17] incubator-slider git commit: SLIDER-622 bypass Haddop APIs
and go to commons IO directory delete if the path is file://
Posted by st...@apache.org.
SLIDER-622 bypass Haddop APIs and go to commons IO directory delete if the path is file://
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f5324897
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f5324897
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f5324897
Branch: refs/heads/releases/slider-0.60
Commit: f5324897d1649f79db9775b2f917282edfb9c810
Parents: 5afa9b3
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:37:28 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:47 2014 +0000
----------------------------------------------------------------------
.../slider/test/YarnMiniClusterTestBase.groovy | 38 +++++++++++++++++---
1 file changed, 34 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5324897/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index ffd3dab..326c217 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.test
import groovy.util.logging.Slf4j
+import org.apache.commons.io.FileUtils
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.conf.Configuration
@@ -550,18 +551,47 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
public void rigorousDelete(
SliderFileSystem sliderFileSystem,
Path path, long timeout) {
+
+ if (path.toUri().scheme == "file") {
+ File dir = new File(path.toUri().getPath());
+ rigorousDelete(dir, timeout)
+ } else {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
+ boolean deleted = false;
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
+ sleep(1000)
+ }
+ }
+ }
+ sliderFileSystem.verifyDirectoryNonexistent(path)
+ }
+
+ /**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dir dir to delete
+ * @param timeout timeout in millis
+ */
+ public void rigorousDelete(File dir, long timeout) {
Duration duration = new Duration(timeout)
duration.start()
- HadoopFS dfs = sliderFileSystem.fileSystem
boolean deleted = false;
while (!deleted && !duration.limitExceeded) {
- dfs.delete(path, true)
- deleted = !dfs.exists(path)
+ FileUtils.deleteQuietly(dir)
+ deleted = !dir.exists()
if (!deleted) {
sleep(1000)
}
}
- sliderFileSystem.verifyDirectoryNonexistent(path)
+ if (!deleted) {
+ // noisy delete raises an IOE
+ FileUtils.deleteDirectory(dir)
+ }
}
/**
[11/17] incubator-slider git commit: SLIDER-622 warn if the directory
is still present after the delete (shutdown race condition problem on NTFS)
Posted by st...@apache.org.
SLIDER-622 warn if the directory is still present after the delete (shutdown race condition problem on NTFS)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d5569711
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d5569711
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d5569711
Branch: refs/heads/releases/slider-0.60
Commit: d5569711eddcd1d4804a8afb42709a5b78f5daee
Parents: b88c775
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 12:58:24 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:28 2014 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d5569711/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index a676fcc..164e2fe 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsAction;
@@ -536,24 +537,23 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
// delete the directory;
- boolean exists = sliderFileSystem.getFileSystem().exists(clusterDirectory);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ boolean exists = fs.exists(clusterDirectory);
if (exists) {
log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
boolean deleted =
- sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
+ fs.delete(clusterDirectory, true);
if (!deleted) {
log.warn("Filesystem returned false from delete() operation");
}
-
- if(!deleteZookeeperNode(clustername)) {
- log.warn("Unable to perform node cleanup in Zookeeper.");
+ if (fs.exists(clusterDirectory)) {
+ log.warn("Failed to delete {}", clusterDirectory);
}
} else {
log.debug("Application Instance {} already destroyed", clustername);
}
-
// rm the registry entry —do not let this block the destroy operations
String registryPath = SliderRegistryUtils.registryPathForInstance(
clustername);
[15/17] incubator-slider git commit: SLIDER-622 add spin-checks for
cluster dir deletion...deals with NTFS problems
Posted by st...@apache.org.
SLIDER-622 add spin-checks for cluster dir deletion...deals with NTFS problems
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5afa9b39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5afa9b39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5afa9b39
Branch: refs/heads/releases/slider-0.60
Commit: 5afa9b39d21d396052d82b5472b344e7903a13db
Parents: 7596387
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:11:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:47 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAMDestroy.groovy | 5 ++++
.../slider/test/YarnMiniClusterTestBase.groovy | 29 ++++++++++----------
2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5afa9b39/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 4a94eb3..d25396d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -121,6 +121,11 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
assert 0 == exitCode
describe "post destroy checks"
+ if (fs.exists(instanceDir)) {
+ log.warn("Destroy operation did not delete $instanceDir")
+ rigorousDelete(sliderFileSystem, instanceDir, 60000)
+ }
+
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
// look up app report and verify exit code is good
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5afa9b39/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 0850d25..ffd3dab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -488,11 +488,13 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
def config = miniCluster.config
if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), config)
- Path clusterDir = new SliderFileSystem(dfs, config).buildClusterDirPath(clustername)
- log.info("deleting customer data at $clusterDir")
+
+ def sliderFileSystem = new SliderFileSystem(dfs, config)
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername)
+ log.info("deleting instance data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- rigorousDelete(dfs, clusterDir, 60000)
+ rigorousDelete(sliderFileSystem, clusterDir, 60000)
}
@@ -543,26 +545,23 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
* Delete with some pauses and backoff; designed to handle slow delete
* operation in windows
* @param dfs
- * @param clusterDir
+ * @param path
*/
public void rigorousDelete(
- HadoopFS dfs, Path clusterDir, long timeout) {
+ SliderFileSystem sliderFileSystem,
+ Path path, long timeout) {
Duration duration = new Duration(timeout)
duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
boolean deleted = false;
- while (!deleted) {
- dfs.delete(clusterDir, true)
- deleted = !dfs.exists(clusterDir)
- if (!deleted ) {
- if (duration.limitExceeded) {
- assert !dfs.exists(
- clusterDir), "delete operation failed —directory in use?"
-
- }
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
sleep(1000)
}
-
}
+ sliderFileSystem.verifyDirectoryNonexistent(path)
}
/**
[17/17] incubator-slider git commit: SLIDER-622 ensuring unique names
for instances in a test
Posted by st...@apache.org.
SLIDER-622 ensuring unique names for instances in a test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56e16c39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56e16c39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56e16c39
Branch: refs/heads/releases/slider-0.60
Commit: 56e16c391524f8799a6f5d56beaafaf968a5c5c8
Parents: f532489
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 14:50:16 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 15:04:47 2014 +0000
----------------------------------------------------------------------
.../main/java/org/apache/slider/core/build/InstanceBuilder.java | 4 +++-
.../org/apache/slider/providers/agent/TestBuildBasicAgent.groovy | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56e16c39/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 669a5ad..cee2663 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -253,7 +253,9 @@ public class InstanceBuilder {
IOException,
SliderException,
LockAcquireFailedException {
- if (!overwrite) coreFS.createClusterDirectories(instancePaths);
+ if (!overwrite) {
+ coreFS.createClusterDirectories(instancePaths);
+ }
ConfPersister persister =
new ConfPersister(coreFS, getInstanceDir());
ConfDirSnapshotAction action = null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56e16c39/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 3f7a432..264d260 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -171,7 +171,7 @@ class TestBuildBasicAgent extends AgentTestBase {
}
try {
- launcher = buildAgentCluster(clustername,
+ launcher = buildAgentCluster(clustername + "-10",
[
(ROLE_NODE): 4,
],
[04/17] incubator-slider git commit: SLIDER-637 rm unneeded test in
TestZKIntegration
Posted by st...@apache.org.
SLIDER-637 rm unneeded test in TestZKIntegration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2bb2323c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2bb2323c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2bb2323c
Branch: refs/heads/develop
Commit: 2bb2323ccd0921da56bfdc502ece5d5b96ee8237
Parents: f9d7d45
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 11 13:50:30 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 11 13:50:30 2014 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/TestZKIntegration.groovy | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2bb2323c/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 0aa1b91..431c49f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -59,16 +59,6 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
zki = null;
}
- @Test
- public void testIntegrationCreate() throws Throwable {
- assertHasZKCluster()
- initZKI()
- String userPath = ZKIntegration.mkSliderUserPath(USER)
- Stat stat = zki.stat(userPath)
- assert stat != null
- log.info("User path $userPath has stat $stat")
- }
-
public ZKIntegration initZKI() {
zki = createZKIntegrationInstance(
getZKBinding(), methodName.methodName, true, false, 5000)