You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/02/10 19:56:41 UTC
[1/6] git commit: ACCUMULO-2332 Rework how address parsing is
currently done by the GC.
Updated Branches:
refs/heads/1.5.1-SNAPSHOT 78ba31d5a -> 30799163e
refs/heads/1.6.0-SNAPSHOT b4b0c0ee6 -> 5a58538e4
refs/heads/master 2ed6fb83c -> 00db80b65
ACCUMULO-2332 Rework how address parsing is currently done by the GC.
Try to reduce the number of creations of an InetSocketAddress and ensure
that an externally addressable hostname/address is placed in zookeeper.
Also, replace 0.0.0.0 with the hostname for the machine when provided
as the address
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/30799163
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/30799163
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/30799163
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 30799163e3eabcf00199989f472c027a3c823132
Parents: 78ba31d
Author: Josh Elser <el...@apache.org>
Authored: Wed Feb 5 23:48:13 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 11:54:09 2014 -0500
----------------------------------------------------------------------
.../server/gc/SimpleGarbageCollector.java | 31 ++++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/30799163/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 5627698..0da84dd 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.server.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -65,6 +66,7 @@ import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.ServerServices;
@@ -128,7 +130,6 @@ public class SimpleGarbageCollector implements Iface {
private FileSystem fs;
private Trash trash = null;
private boolean safemode = false, offline = false, verbose = false;
- private String address = "localhost";
private ZooLock lock;
private Key continueKey = null;
@@ -157,11 +158,19 @@ public class SimpleGarbageCollector implements Iface {
if (opts.verbose)
gc.setVerbose();
if (opts.address != null)
- gc.useAddress(opts.address);
+ address = opts.address;
+
+ String hostname = address;
+
+ // Catch a few edge cases where we might get a non-useful address
+ if ("0.0.0.0".equals(address)) {
+ // Get a better hostname for this system
+ hostname = InetAddress.getLocalHost().getHostName();
+ }
gc.init(fs, instance, SecurityConstants.getSystemCredentials(), serverConf.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE));
- Accumulo.enableTracing(address, "gc");
- gc.run();
+ Accumulo.enableTracing(hostname, "gc");
+ gc.run(hostname);
}
public SimpleGarbageCollector() {}
@@ -178,10 +187,6 @@ public class SimpleGarbageCollector implements Iface {
this.verbose = true;
}
- public void useAddress(String address) {
- this.address = address;
- }
-
public void init(FileSystem fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException {
this.fs = TraceFileSystem.wrap(fs);
this.credentials = credentials;
@@ -202,14 +207,14 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private void run() {
+ private void run(String hostname) {
long tStart, tStop;
// Sleep for an initial period, giving the master time to start up and
// old data files to be unused
if (!offline) {
try {
- getZooLock(startStatsService());
+ getZooLock(startStatsService(hostname));
} catch (Exception ex) {
log.error(ex, ex);
System.exit(1);
@@ -383,7 +388,7 @@ public class SimpleGarbageCollector implements Iface {
}
private void getZooLock(InetSocketAddress addr) throws KeeperException, InterruptedException {
- String address = addr.getHostName() + ":" + addr.getPort();
+ String address = AddressUtil.toString(addr);
String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
LockWatcher lockWatcher = new LockWatcher() {
@@ -413,7 +418,7 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private InetSocketAddress startStatsService() throws UnknownHostException {
+ private InetSocketAddress startStatsService(String hostname) throws UnknownHostException {
Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this));
int port = instance.getConfiguration().getPort(Property.GC_PORT);
long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
@@ -423,7 +428,7 @@ public class SimpleGarbageCollector implements Iface {
log.fatal(ex, ex);
throw new RuntimeException(ex);
}
- return new InetSocketAddress(Accumulo.getLocalAddress(new String[] {"--address", address}), port);
+ return new InetSocketAddress(hostname, port);
}
/**
[6/6] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/00db80b6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/00db80b6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/00db80b6
Branch: refs/heads/master
Commit: 00db80b6573b4598b6bc964df73ee70fb6b037e3
Parents: 2ed6fb8 5a58538
Author: Josh Elser <el...@apache.org>
Authored: Mon Feb 10 13:56:15 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 13:56:15 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[2/6] git commit: ACCUMULO-2332 Rework how address parsing is
currently done by the GC.
Posted by el...@apache.org.
ACCUMULO-2332 Rework how address parsing is currently done by the GC.
Try to reduce the number of creations of an InetSocketAddress and ensure
that an externally addressable hostname/address is placed in zookeeper.
Also, replace 0.0.0.0 with the hostname for the machine when provided
as the address
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/30799163
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/30799163
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/30799163
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 30799163e3eabcf00199989f472c027a3c823132
Parents: 78ba31d
Author: Josh Elser <el...@apache.org>
Authored: Wed Feb 5 23:48:13 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 11:54:09 2014 -0500
----------------------------------------------------------------------
.../server/gc/SimpleGarbageCollector.java | 31 ++++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/30799163/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 5627698..0da84dd 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.server.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -65,6 +66,7 @@ import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.ServerServices;
@@ -128,7 +130,6 @@ public class SimpleGarbageCollector implements Iface {
private FileSystem fs;
private Trash trash = null;
private boolean safemode = false, offline = false, verbose = false;
- private String address = "localhost";
private ZooLock lock;
private Key continueKey = null;
@@ -157,11 +158,19 @@ public class SimpleGarbageCollector implements Iface {
if (opts.verbose)
gc.setVerbose();
if (opts.address != null)
- gc.useAddress(opts.address);
+ address = opts.address;
+
+ String hostname = address;
+
+ // Catch a few edge cases where we might get a non-useful address
+ if ("0.0.0.0".equals(address)) {
+ // Get a better hostname for this system
+ hostname = InetAddress.getLocalHost().getHostName();
+ }
gc.init(fs, instance, SecurityConstants.getSystemCredentials(), serverConf.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE));
- Accumulo.enableTracing(address, "gc");
- gc.run();
+ Accumulo.enableTracing(hostname, "gc");
+ gc.run(hostname);
}
public SimpleGarbageCollector() {}
@@ -178,10 +187,6 @@ public class SimpleGarbageCollector implements Iface {
this.verbose = true;
}
- public void useAddress(String address) {
- this.address = address;
- }
-
public void init(FileSystem fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException {
this.fs = TraceFileSystem.wrap(fs);
this.credentials = credentials;
@@ -202,14 +207,14 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private void run() {
+ private void run(String hostname) {
long tStart, tStop;
// Sleep for an initial period, giving the master time to start up and
// old data files to be unused
if (!offline) {
try {
- getZooLock(startStatsService());
+ getZooLock(startStatsService(hostname));
} catch (Exception ex) {
log.error(ex, ex);
System.exit(1);
@@ -383,7 +388,7 @@ public class SimpleGarbageCollector implements Iface {
}
private void getZooLock(InetSocketAddress addr) throws KeeperException, InterruptedException {
- String address = addr.getHostName() + ":" + addr.getPort();
+ String address = AddressUtil.toString(addr);
String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
LockWatcher lockWatcher = new LockWatcher() {
@@ -413,7 +418,7 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private InetSocketAddress startStatsService() throws UnknownHostException {
+ private InetSocketAddress startStatsService(String hostname) throws UnknownHostException {
Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this));
int port = instance.getConfiguration().getPort(Property.GC_PORT);
long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
@@ -423,7 +428,7 @@ public class SimpleGarbageCollector implements Iface {
log.fatal(ex, ex);
throw new RuntimeException(ex);
}
- return new InetSocketAddress(Accumulo.getLocalAddress(new String[] {"--address", address}), port);
+ return new InetSocketAddress(hostname, port);
}
/**
[4/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5a58538e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5a58538e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5a58538e
Branch: refs/heads/master
Commit: 5a58538e41252c5c455219b8fcb29326c8b7631b
Parents: b4b0c0e 3079916
Author: Josh Elser <el...@apache.org>
Authored: Mon Feb 10 13:56:07 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 13:56:07 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[3/6] git commit: ACCUMULO-2332 Rework how address parsing is
currently done by the GC.
Posted by el...@apache.org.
ACCUMULO-2332 Rework how address parsing is currently done by the GC.
Try to reduce the number of creations of an InetSocketAddress and ensure
that an externally addressable hostname/address is placed in zookeeper.
Also, replace 0.0.0.0 with the hostname for the machine when provided
as the address
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/30799163
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/30799163
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/30799163
Branch: refs/heads/master
Commit: 30799163e3eabcf00199989f472c027a3c823132
Parents: 78ba31d
Author: Josh Elser <el...@apache.org>
Authored: Wed Feb 5 23:48:13 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 11:54:09 2014 -0500
----------------------------------------------------------------------
.../server/gc/SimpleGarbageCollector.java | 31 ++++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/30799163/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 5627698..0da84dd 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.server.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -65,6 +66,7 @@ import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.ServerServices;
@@ -128,7 +130,6 @@ public class SimpleGarbageCollector implements Iface {
private FileSystem fs;
private Trash trash = null;
private boolean safemode = false, offline = false, verbose = false;
- private String address = "localhost";
private ZooLock lock;
private Key continueKey = null;
@@ -157,11 +158,19 @@ public class SimpleGarbageCollector implements Iface {
if (opts.verbose)
gc.setVerbose();
if (opts.address != null)
- gc.useAddress(opts.address);
+ address = opts.address;
+
+ String hostname = address;
+
+ // Catch a few edge cases where we might get a non-useful address
+ if ("0.0.0.0".equals(address)) {
+ // Get a better hostname for this system
+ hostname = InetAddress.getLocalHost().getHostName();
+ }
gc.init(fs, instance, SecurityConstants.getSystemCredentials(), serverConf.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE));
- Accumulo.enableTracing(address, "gc");
- gc.run();
+ Accumulo.enableTracing(hostname, "gc");
+ gc.run(hostname);
}
public SimpleGarbageCollector() {}
@@ -178,10 +187,6 @@ public class SimpleGarbageCollector implements Iface {
this.verbose = true;
}
- public void useAddress(String address) {
- this.address = address;
- }
-
public void init(FileSystem fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException {
this.fs = TraceFileSystem.wrap(fs);
this.credentials = credentials;
@@ -202,14 +207,14 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private void run() {
+ private void run(String hostname) {
long tStart, tStop;
// Sleep for an initial period, giving the master time to start up and
// old data files to be unused
if (!offline) {
try {
- getZooLock(startStatsService());
+ getZooLock(startStatsService(hostname));
} catch (Exception ex) {
log.error(ex, ex);
System.exit(1);
@@ -383,7 +388,7 @@ public class SimpleGarbageCollector implements Iface {
}
private void getZooLock(InetSocketAddress addr) throws KeeperException, InterruptedException {
- String address = addr.getHostName() + ":" + addr.getPort();
+ String address = AddressUtil.toString(addr);
String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
LockWatcher lockWatcher = new LockWatcher() {
@@ -413,7 +418,7 @@ public class SimpleGarbageCollector implements Iface {
}
}
- private InetSocketAddress startStatsService() throws UnknownHostException {
+ private InetSocketAddress startStatsService(String hostname) throws UnknownHostException {
Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this));
int port = instance.getConfiguration().getPort(Property.GC_PORT);
long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
@@ -423,7 +428,7 @@ public class SimpleGarbageCollector implements Iface {
log.fatal(ex, ex);
throw new RuntimeException(ex);
}
- return new InetSocketAddress(Accumulo.getLocalAddress(new String[] {"--address", address}), port);
+ return new InetSocketAddress(hostname, port);
}
/**
[5/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5a58538e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5a58538e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5a58538e
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 5a58538e41252c5c455219b8fcb29326c8b7631b
Parents: b4b0c0e 3079916
Author: Josh Elser <el...@apache.org>
Authored: Mon Feb 10 13:56:07 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Feb 10 13:56:07 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------