You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2014/01/13 17:18:55 UTC
git commit: ACCUMULO-2163 fix GC IT test and start() only run gc if
not already started
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 0ab78c648 -> b54266a8a
ACCUMULO-2163 fix GC IT test and start() only run gc if not already started
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b54266a8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b54266a8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b54266a8
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: b54266a8a4da7a14d77416f9d0acf3a937e4bb60
Parents: 0ab78c6
Author: Keith Turner <kt...@apache.org>
Authored: Fri Jan 10 20:28:08 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Mon Jan 13 10:52:15 2014 -0500
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 5 +-
.../test/functional/GarbageCollectorIT.java | 122 +++++++++++--------
2 files changed, 74 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b54266a8/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 09af10b..a2c34d4 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -480,7 +480,10 @@ public class MiniAccumuloCluster {
if (masterProcess == null) {
masterProcess = _exec(Master.class, ServerType.MASTER);
}
- gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+
+ if (gcProcess == null) {
+ gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+ }
}
private List<String> buildRemoteDebugParams(int port) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b54266a8/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index b813ca9..44c57c1 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.test.functional;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.util.Collections;
@@ -31,6 +32,7 @@ import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -44,9 +46,11 @@ import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.gc.SimpleGarbageCollector;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.ProcessNotFoundException;
import org.apache.accumulo.minicluster.ProcessReference;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -56,6 +60,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.junit.Assert;
import org.junit.Test;
@@ -76,8 +81,24 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
cfg.useMiniDFS();
}
+ private void killMacGc() throws ProcessNotFoundException, InterruptedException, KeeperException {
+ // kill gc started by MAC
+ getCluster().killProcess(ServerType.GARBAGE_COLLECTOR, getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
+ // delete lock in zookeeper if there, this will allow next GC to start quickly
+ String path = ZooUtil.getRoot(new ZooKeeperInstance(getCluster().getClientConfig())) + Constants.ZGC_LOCK;
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ try {
+ ZooLock.deleteLock(zk, path);
+ } catch (IllegalStateException e) {
+
+ }
+
+ assertNull(getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR));
+ }
+
@Test(timeout = 4 * 60 * 1000)
public void gcTest() throws Exception {
+ killMacGc();
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
@@ -95,19 +116,19 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
break;
before = more;
}
- Process gc = cluster.exec(SimpleGarbageCollector.class);
- try {
- UtilWaitThread.sleep(10 * 1000);
- int after = countFiles();
- VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- assertTrue(after < before);
- } finally {
- gc.destroy();
- }
+
+ // restart GC
+ getCluster().start();
+ UtilWaitThread.sleep(10 * 1000);
+ int after = countFiles();
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ assertTrue(after < before);
}
@Test(timeout = 4 * 60 * 1000)
public void gcLotsOfCandidatesIT() throws Exception {
+ killMacGc();
+
log.info("Filling metadata table with bogus delete flags");
Connector c = getConnector();
addEntries(c, new BatchWriterOpts());
@@ -121,14 +142,15 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
@Test(timeout = 20 * 60 * 1000)
public void dontGCRootLog() throws Exception {
+ killMacGc();
// dirty metadata
Connector c = getConnector();
String table = getTableNames(1)[0];
c.tableOperations().create(table);
// let gc run for a bit
- Process gc = cluster.exec(SimpleGarbageCollector.class);
+ cluster.start();
UtilWaitThread.sleep(20 * 1000);
- gc.destroy();
+ killMacGc();
// kill tservers
for (ProcessReference ref : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, ref);
@@ -145,52 +167,48 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
@Test(timeout = 60 * 1000)
public void testProperPortAdvertisement() throws Exception {
- Process gc = cluster.exec(SimpleGarbageCollector.class);
+
Connector conn = getConnector();
Instance instance = conn.getInstance();
-
- try {
- ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
- String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
- for (int i = 0; i < 5; i++) {
- List<String> locks;
- try {
- locks = zk.getChildren(path, null);
- } catch (NoNodeException e ) {
- Thread.sleep(5000);
- continue;
- }
-
- if (locks != null && locks.size() > 0) {
- Collections.sort(locks);
-
- String lockPath = path + "/" + locks.get(0);
-
- String gcLoc = new String(zk.getData(lockPath, null));
-
- Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc, gcLoc.startsWith(Service.GC_CLIENT.name()));
- int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
- Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc, -1, loc);
- String addr = gcLoc.substring(loc + 1);
-
- int addrSplit = addr.indexOf(':');
- Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
-
- String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
- // We shouldn't have the "bindall" address in zk
- Assert.assertNotEquals("0.0.0.0", host);
- // Nor should we have the "random port" in zk
- Assert.assertNotEquals(0, Integer.parseInt(port));
- return;
- }
-
+
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ for (int i = 0; i < 5; i++) {
+ List<String> locks;
+ try {
+ locks = zk.getChildren(path, null);
+ } catch (NoNodeException e) {
Thread.sleep(5000);
+ continue;
+ }
+
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
+
+ String lockPath = path + "/" + locks.get(0);
+
+ String gcLoc = new String(zk.getData(lockPath, null));
+
+ Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc, gcLoc.startsWith(Service.GC_CLIENT.name()));
+ int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
+ Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc, -1, loc);
+ String addr = gcLoc.substring(loc + 1);
+
+ int addrSplit = addr.indexOf(':');
+ Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
+
+ String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
+ // We shouldn't have the "bindall" address in zk
+ Assert.assertNotEquals("0.0.0.0", host);
+ // Nor should we have the "random port" in zk
+ Assert.assertNotEquals(0, Integer.parseInt(port));
+ return;
}
-
- Assert.fail("Could not find advertised GC address");
- } finally {
- gc.destroy();
+
+ Thread.sleep(5000);
}
+
+ Assert.fail("Could not find advertised GC address");
}
private int countFiles() throws Exception {