You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/07 03:44:47 UTC
[1/2] git commit: ACCUMULO-2142 Return the HostAndPort from creating
the thrift server
Updated Branches:
refs/heads/master 33a094de4 -> ac28b2253
ACCUMULO-2142 Return the HostAndPort from creating the thrift server
The GC returned the original HostAndPort object instead of the one
returned by TServerUtils, and ultimately the HsHaServer. The problem
here is that TServerUtils is doing some magic to unwind things like
0.0.0.0 as an addr and a port of 0.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/da9e0f40
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/da9e0f40
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/da9e0f40
Branch: refs/heads/master
Commit: da9e0f40a34b339eaa23093fce1603c98de0d5d9
Parents: ea8fe54
Author: Josh Elser <el...@apache.org>
Authored: Mon Jan 6 20:08:27 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jan 6 21:38:35 2014 -0500
----------------------------------------------------------------------
.../accumulo/gc/SimpleGarbageCollector.java | 4 +-
.../test/functional/GarbageCollectorIT.java | 78 ++++++++++++++++++--
2 files changed, 74 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da9e0f40/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9286a37..fc24010 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -535,13 +535,13 @@ public class SimpleGarbageCollector implements Iface {
int port = ServerConfiguration.getSystemConfiguration(instance).getPort(Property.GC_PORT);
long maxMessageSize = ServerConfiguration.getSystemConfiguration(instance).getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
HostAndPort result = HostAndPort.fromParts(opts.getAddress(), port);
+ log.debug("Starting garbage collector listening on " + result);
try {
- port = TServerUtils.startTServer(result, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize).address.getPort();
+ return TServerUtils.startTServer(result, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize).address;
} catch (Exception ex) {
log.fatal(ex, ex);
throw new RuntimeException(ex);
}
- return result;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da9e0f40/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index df1a3f1..b813ca9 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -18,14 +18,18 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertTrue;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -36,27 +40,36 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.gc.SimpleGarbageCollector;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.minicluster.ProcessReference;
import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.junit.Assert;
import org.junit.Test;
public class GarbageCollectorIT extends ConfigurableMacIT {
-
+ private static final String OUR_SECRET = "itsreallysecret";
+
@Override
public void configure(MiniAccumuloConfig cfg) {
Map<String,String> settings = new HashMap<String,String>();
+ settings.put(Property.INSTANCE_SECRET.getKey(), OUR_SECRET);
settings.put(Property.GC_CYCLE_START.getKey(), "1");
settings.put(Property.GC_CYCLE_DELAY.getKey(), "1");
+ settings.put(Property.GC_PORT.getKey(), "0");
settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
cfg.setSiteConfig(settings);
@@ -83,11 +96,14 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
before = more;
}
Process gc = cluster.exec(SimpleGarbageCollector.class);
- UtilWaitThread.sleep(10 * 1000);
- int after = countFiles();
- VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- assertTrue(after < before);
- gc.destroy();
+ try {
+ UtilWaitThread.sleep(10 * 1000);
+ int after = countFiles();
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ assertTrue(after < before);
+ } finally {
+ gc.destroy();
+ }
}
@Test(timeout = 4 * 60 * 1000)
@@ -127,6 +143,56 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
}
}
+ @Test(timeout = 60 * 1000)
+ public void testProperPortAdvertisement() throws Exception {
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ Connector conn = getConnector();
+ Instance instance = conn.getInstance();
+
+ try {
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ for (int i = 0; i < 5; i++) {
+ List<String> locks;
+ try {
+ locks = zk.getChildren(path, null);
+ } catch (NoNodeException e ) {
+ Thread.sleep(5000);
+ continue;
+ }
+
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
+
+ String lockPath = path + "/" + locks.get(0);
+
+ String gcLoc = new String(zk.getData(lockPath, null));
+
+ Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc, gcLoc.startsWith(Service.GC_CLIENT.name()));
+ int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
+ Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc, -1, loc);
+ String addr = gcLoc.substring(loc + 1);
+
+ int addrSplit = addr.indexOf(':');
+ Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
+
+ String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
+ // We shouldn't have the "bindall" address in zk
+ Assert.assertNotEquals("0.0.0.0", host);
+ // Nor should we have the "random port" in zk
+ Assert.assertNotEquals(0, Integer.parseInt(port));
+ return;
+ }
+
+ Thread.sleep(5000);
+ }
+
+ Assert.fail("Could not find advertised GC address");
+ } finally {
+ gc.destroy();
+ }
+ }
+
private int countFiles() throws Exception {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
int result = 0;
[2/2] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ac28b225
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ac28b225
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ac28b225
Branch: refs/heads/master
Commit: ac28b2253f7165c088472de55562a79e4fce8cf0
Parents: 33a094d da9e0f4
Author: Josh Elser <el...@apache.org>
Authored: Mon Jan 6 21:44:34 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jan 6 21:44:34 2014 -0500
----------------------------------------------------------------------
.../accumulo/gc/SimpleGarbageCollector.java | 4 +-
.../test/functional/GarbageCollectorIT.java | 78 ++++++++++++++++++--
2 files changed, 74 insertions(+), 8 deletions(-)
----------------------------------------------------------------------