You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/04/30 20:18:35 UTC
svn commit: r1591434 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/mapreduce/
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop...
Author: liyin
Date: Wed Apr 30 18:18:34 2014
New Revision: 1591434
URL: http://svn.apache.org/r1591434
Log:
[HBASE-11097] Make HBase compatible with IPv6
Author: everyoung
Summary:
Wiki page at https://our.intern.facebook.com/intern/wiki/index.php/HBase/HBase_IPv6_Compatible
1. Add TestIPv6HServerInfo and make HServerInfo recognize IPv6 address,
using HostAndPort & InternetDomainNames to verify hostname and port
instead of using regular expression for ServerManager and HServerInfo.
2. Add HQuorumPeerConfig according to trunk version of org.apache.zookeeper.server.quorum.QuorumPeerConfig,
fix Zookeeper IPv6 connection problem following the Zookeeper convention at https://phabricator.fb.com/D1226868.
The IPv6 address in zoo.cfg must be enclosed with []. So the parsing and createZK parameters needs to be changed.
3. Fix Log naming issues, serverName.replace(":", "%3A") to create LOG names in HLog.java,
ZKSplitLog.encode(workername) to create tmp LOG name in SplitLogMananer and SplitLogWorker (ZKSplitLog.decode to recover).
Not using encode(serverName) for HLog.java because it will also convert ',' and not compatible with IPv4 addresses.
4. Mark ReplicationLogCleaner and ReplicationZookeeperWrapper deprecated as Jiqing indicated.
5. Add config to let local machine use IPv6 address if possible in HMaster and HRegionServer.
6. fix TableInputFormatBase$reverseDNS to use appropriate API.
7. Use IPv6AddressTruncationMapping instead of IPv4AddressTruncationMapping in RackManager.java.
7. The IPv6 address for HBase can be the raw string or enclosed with [].
Test Plan: All unit tests and new tests
Reviewers: aaiyer, liyintang, akshay, manukranthk
Reviewed By: manukranthk
CC: hbase-dev@, manukranthk
Differential Revision: https://phabricator.fb.com/D1265426
Task ID: 3586081
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeerConfig.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestIPv6HServerInfo.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RackManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionChecker.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerInfo.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerInfo.java Wed Apr 30 18:18:34 2014
@@ -22,13 +22,22 @@ package org.apache.hadoop.hbase;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.SortedMap;
import java.util.concurrent.ConcurrentSkipListMap;
+import com.google.common.net.HostAndPort;
+import com.google.common.net.InetAddresses;
+import com.google.common.net.InternetDomainName;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
@@ -65,9 +74,8 @@ public class HServerInfo implements Writ
*/
static final String SERVERNAME_SEPARATOR = ",";
- private static final Pattern SERVER_NAME_RE = Pattern.compile(
- "^[0-9a-zA-Z.-]+" + SERVERNAME_SEPARATOR +
- "[0-9]{1," + String.valueOf(0xffff).length() + "}" + SERVERNAME_SEPARATOR +
+ // RE to match startcode, hostname and port will be parsed by HostAndPort and InternetDomainName
+ private static final Pattern SERVER_START_CODE_RE = Pattern.compile(
"-?[0-9]{1," + String.valueOf(Long.MAX_VALUE).length() + "}");
private HServerAddress serverAddress;
@@ -231,7 +239,8 @@ public class HServerInfo implements Writ
public static synchronized String getServerName(final String hostAndPort,
final long startcode) {
- int index = hostAndPort.indexOf(":");
+ // use lastIndexOf to comply with IPv6 addresses
+ int index = hostAndPort.lastIndexOf(":");
if (index <= 0) throw new IllegalArgumentException("Expected <hostname> ':' <port>");
return getServerName(hostAndPort.substring(0, index),
Integer.parseInt(hostAndPort.substring(index + 1)), startcode);
@@ -376,11 +385,95 @@ public class HServerInfo implements Writ
}
public static boolean isValidServerName(String serverName) {
- return SERVER_NAME_RE.matcher(serverName).matches();
+ String[] parts = serverName.split(SERVERNAME_SEPARATOR);
+ if (parts.length == 3 && isValidHostAndPort(parts[0] + ":" + parts[1])) {
+ // check the other parts
+ return SERVER_START_CODE_RE.matcher(parts[2]).matches();
+ }
+ return false;
+ }
+
+ public static boolean isValidHostAndPort(String hostPort) {
+ try {
+ int portIndex = hostPort.lastIndexOf(":");
+ // HostAndPort.fromString will treat multiple ":" as host-only string
+ // need to separate port number and use HostAndPort.fromParts
+ HostAndPort hostAndPort = HostAndPort.fromParts(hostPort.substring(0, portIndex),
+ Integer.valueOf(hostPort.substring(portIndex+1)));
+ // HostAndPort will guarantee port validity
+ // check hostname sanity
+ try {
+ String hostname = hostAndPort.getHostText();
+ // chop off zone index part b/c InetAddresses does not parse that
+ if (hostname.contains("%")) {
+ hostname = hostname.substring(0, hostname.indexOf("%"));
+ }
+ InetAddresses.forString(hostname);
+ } catch (IllegalArgumentException e) {
+ // not an ip string
+ try {
+ // not using HostSpecifier because it requires a public suffix
+ InternetDomainName.from(hostAndPort.getHostText());
+ } catch (IllegalArgumentException e1) {
+ return false;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
}
public static HServerAddress getAddress(HServerInfo hsi) {
return hsi != null ? hsi.getServerAddress() : null;
}
+ /**
+ * If the serverName represents the local machine and
+ * there is an global ipv6 address for the host
+ * return the global ipv6 address
+ * otherwise return the serverName itself
+ * @param serverName
+ * @return
+ * @throws SocketException
+ */
+ public static String getIPv6AddrIfLocalMachine(String serverName) throws SocketException {
+ // if it is null or already an IPv6 address, return it
+ if (serverName == null || serverName.contains(":")) {
+ return serverName;
+ }
+ // For all nics get all hostnames and IPs
+ List<String> ips = new ArrayList<String>();
+ List<String> ip6s = new ArrayList<String>();
+ Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
+ while(nics.hasMoreElements()) {
+ Enumeration<?> rawAdrs =
+ ((NetworkInterface)nics.nextElement()).getInetAddresses();
+ while(rawAdrs.hasMoreElements()) {
+ InetAddress inet = (InetAddress) rawAdrs.nextElement();
+ ips.add(inet.getHostName());
+ String hostAddr = inet.getHostAddress();
+ ips.add(hostAddr);
+ // Record global IPv6 address
+ if (hostAddr.contains(":") && !inet.isLinkLocalAddress()) {
+ if (hostAddr.contains("%")) {
+ ips.add(hostAddr.substring(0, hostAddr.indexOf("%")));
+ ip6s.add(hostAddr.substring(0, hostAddr.indexOf("%")));
+ } else {
+ ip6s.add(hostAddr);
+ }
+ }
+ }
+ }
+ // if on local machine and has global ipv6 address,
+ // use IPv6 address
+ if (!ip6s.isEmpty() &&
+ (serverName.equals("localhost") || ips.contains(serverName))) {
+ return ip6s.get(0);
+ }
+ // return the original serverName
+ return serverName;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java Wed Apr 30 18:18:34 2014
@@ -333,7 +333,7 @@ extends InputFormat<ImmutableBytesWritab
}
private static String reverseDNS(InetAddress ipAddress)
- throws UnknownHostException {
+ throws UnknownHostException {
String hostName = reverseDNSCacheMap.get(ipAddress);
if (hostName == null) {
// DNS.reverseDns(ipAddress, nameServer) is abandoned
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed Apr 30 18:18:34 2014
@@ -29,6 +29,7 @@ import java.lang.management.ManagementFa
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
+import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -263,6 +264,9 @@ public class HMaster extends HasThread i
private long schemaChangeTryLockTimeoutMs;
+ /** Configuration variable to prefer IPv6 address for HMaster */
+ public static final String HMASTER_PREFER_IPV6_Address = "hmaster.prefer.ipv6.address";
+
/**
* Constructor
* @param conf configuration
@@ -273,7 +277,7 @@ public class HMaster extends HasThread i
// Get my address. So what's the difference between the temp a and address?
// address.toString() will always produce an IP address as the hostname.
- // a.toStrng() can have the canonical-name of the host as the hostname.
+ // a.toString() can have the canonical-name of the host as the hostname.
// If the configured master port is 0, then a will bind it to an
// ephemeral port first by starting the rpc server.
HServerAddress a = new HServerAddress(getMyAddress(this.conf));
@@ -459,7 +463,7 @@ public class HMaster extends HasThread i
/**
* @return true if successfully became primary master
*/
- private boolean waitToBecomePrimary() {
+ private boolean waitToBecomePrimary() throws SocketException {
if (!this.zkMasterAddressWatcher.writeAddressToZooKeeper(this.address,
true)) {
LOG.info("Failed to write master address to ZooKeeper, not starting (" +
@@ -661,10 +665,14 @@ public class HMaster extends HasThread i
* @throws UnknownHostException
*/
private static String getMyAddress(final Configuration c)
- throws UnknownHostException {
+ throws UnknownHostException, SocketException {
// Find out our address up in DNS.
String s = DNS.getDefaultHost(c.get("hbase.master.dns.interface","default"),
c.get("hbase.master.dns.nameserver","default"));
+ if (preferIpv6AddressForMaster(c)) {
+ // Use IPv6 address if possible.
+ s = HServerInfo.getIPv6AddrIfLocalMachine(s);
+ }
s += ":" + c.get(HConstants.MASTER_PORT,
Integer.toString(HConstants.DEFAULT_MASTER_PORT));
return s;
@@ -796,8 +804,13 @@ public class HMaster extends HasThread i
/** Main processing loop */
@Override
public void run() {
- if (!waitToBecomePrimary()) {
- LOG.info("Failed to become primary -- not starting");
+ try {
+ if (!waitToBecomePrimary()) {
+ LOG.error("Failed to become primary -- not starting");
+ return;
+ }
+ } catch (SocketException e) {
+ LOG.error(e);
return;
}
@@ -2434,5 +2447,13 @@ public class HMaster extends HasThread i
@Override
public void close() throws Exception {
}
+
+ /**
+ * @param conf
+ * @return true if the hmaster.prefer.ipv6.address is set
+ */
+ public static boolean preferIpv6AddressForMaster(Configuration conf) {
+ return conf != null ? conf.getBoolean(HMASTER_PREFER_IPV6_Address, false) : false;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RackManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RackManager.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RackManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RackManager.java Wed Apr 30 18:18:34 2014
@@ -10,27 +10,28 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.IPv4AddressTruncationMapping;
+import org.apache.hadoop.net.IPv6AddressTruncationMapping;
public class RackManager {
static final Log LOG = LogFactory.getLog(RackManager.class);
private DNSToSwitchMapping switchMapping;
public RackManager(Configuration conf) {
+ // using IPv6AddressTruncationMapping instead to handle both IPv4 and IPv6 addresses
Class<DNSToSwitchMapping> clz = (Class<DNSToSwitchMapping>)
conf.getClass("hbase.util.ip.to.rack.determiner",
- IPv4AddressTruncationMapping.class);
+ IPv6AddressTruncationMapping.class);
try {
switchMapping = clz.newInstance();
} catch (InstantiationException e) {
- LOG.warn("using IPv4AddressTruncationMapping, failed to instantiate " +
+ LOG.warn("using IPv6AddressTruncationMapping, failed to instantiate " +
clz.getName(), e);
} catch (IllegalAccessException e) {
- LOG.warn("using IPv4AddressTruncationMapping, failed to instantiate " +
+ LOG.warn("using IPv6AddressTruncationMapping, failed to instantiate " +
clz.getName(), e);
}
if (switchMapping == null) {
- switchMapping = new IPv4AddressTruncationMapping();
+ switchMapping = new IPv6AddressTruncationMapping();
}
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionChecker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionChecker.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionChecker.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionChecker.java Wed Apr 30 18:18:34 2014
@@ -12,6 +12,7 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
+import java.net.SocketException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
@@ -65,7 +66,7 @@ public class RegionChecker {
return regionCheckerEnabled;
}
- public RegionChecker(final HMaster master) {
+ public RegionChecker(final HMaster master) throws SocketException {
Configuration conf = master.getConfiguration();
this.regionCheckerEnabled = conf.getBoolean(HConstants.REGION_CHECKER_ENABLED, HConstants.DEFAULT_REGION_CHECKER_ENABLED);
this.master = master;
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Wed Apr 30 18:18:34 2014
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
+import java.net.SocketException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -32,7 +33,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -69,9 +69,6 @@ public class ServerManager implements Co
private static final Log LOG =
LogFactory.getLog(ServerManager.class.getName());
- public static final Pattern HOST_PORT_RE =
- Pattern.compile("^[0-9a-zA-Z-_.]+:[0-9]{1,5}$");
-
private final AtomicInteger quiescedServers = new AtomicInteger(0);
// The map of known server names to server info
@@ -209,7 +206,7 @@ public class ServerManager implements Co
* Constructor.
* @param master
*/
- public ServerManager(HMaster master) {
+ public ServerManager(HMaster master) throws SocketException {
this.master = master;
Configuration c = master.getConfiguration();
this.nobalancingCount = c.getInt("hbase.regions.nobalancing.count", 4);
@@ -1193,7 +1190,7 @@ public class ServerManager implements Co
}
public static void blacklistRSHostPort(String hostPort) {
- if (!HOST_PORT_RE.matcher(hostPort).matches()) {
+ if (!HServerInfo.isValidHostAndPort(hostPort)) {
throw new IllegalArgumentException("host:port pair expected but got " +
hostPort);
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Wed Apr 30 18:18:34 2014
@@ -133,7 +133,7 @@ public class SplitLogManager implements
@Override
public Status finish(String workerName, String logfile) {
String tmpname =
- ZKSplitLog.getSplitLogDirTmpComponent(workerName, logfile);
+ ZKSplitLog.getSplitLogDirTmpComponent(ZKSplitLog.encode(workerName), logfile);
try {
HLogSplitter.moveRecoveredEditsFromTemp(tmpname, logfile, conf);
} catch (IOException e) {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed Apr 30 18:18:34 2014
@@ -424,6 +424,10 @@ public class HRegionServer implements HR
// It should always be 0 in production environment!
public static volatile long openRegionDelay = 0;
+ /** Configuration variable to prefer IPv6 address for HRegionServer */
+ public static final String HREGIONSERVER_PREFER_IPV6_Address =
+ "hregionserver.prefer.ipv6.address";
+
/**
* Starts a HRegionServer at the default location. This should be followed
* by a call to the initialize() method on the HRegionServer object, to start
@@ -435,9 +439,15 @@ public class HRegionServer implements HR
public HRegionServer(Configuration conf) throws IOException {
LOG.debug("Region server configured with ZK client port "
+ conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
- machineName = DNS.getDefaultHost(
+ String defaultHost = DNS.getDefaultHost(
conf.get("hbase.regionserver.dns.interface","default"),
conf.get("hbase.regionserver.dns.nameserver","default"));
+ if (preferIpv6AddressForRegionServer(conf)) {
+ // Use IPv6 address for HRegionServer.
+ machineName = HServerInfo.getIPv6AddrIfLocalMachine(defaultHost);
+ } else {
+ machineName = defaultHost;
+ }
String addressStr = machineName + ":" +
conf.get(HConstants.REGIONSERVER_PORT,
Integer.toString(HConstants.DEFAULT_REGIONSERVER_PORT));
@@ -4198,4 +4208,12 @@ public class HRegionServer implements HR
public void requestCompaction(HRegionIf r, String why) {
this.compactSplitThread.requestCompaction((HRegion) r, why);
}
+
+ /**
+ * @param conf
+ * @return true if the hregionserver.prefer.ipv6.address is set
+ */
+ public static boolean preferIpv6AddressForRegionServer(Configuration conf) {
+ return conf != null ? conf.getBoolean(HREGIONSERVER_PREFER_IPV6_Address, false) : false;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java Wed Apr 30 18:18:34 2014
@@ -155,7 +155,7 @@ public class SplitLogWorker implements R
return Status.ERR;
}
String tmpname = ZKSplitLog.getSplitLogDirTmpComponent(
- workerName, filename);
+ ZKSplitLog.encode(workerName), filename);
if (!HLogSplitter.splitLogFileToTemp(rootdir, tmpname,
st, fs, conf, p, logCloseThreadPool, masterRef.get())) {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Wed Apr 30 18:18:34 2014
@@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.util.Runt
import org.apache.hadoop.hbase.util.RuntimeHaltAbortStrategy;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
@@ -1852,7 +1853,8 @@ public class HLog implements Syncable {
public static String getHLogDirectoryName(String serverName) {
StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
dirName.append("/");
- dirName.append(serverName);
+ // replace ":" with "%3A"
+ dirName.append(serverName.replace(":", "%3A"));
return dirName.toString();
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java Wed Apr 30 18:18:34 2014
@@ -70,6 +70,7 @@ import org.apache.zookeeper.Watcher;
* ...
* </pre>
*/
+@Deprecated
public class ReplicationZookeeperWrapper {
private static final Log LOG =
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java Wed Apr 30 18:18:34 2014
@@ -41,6 +41,7 @@ import java.util.concurrent.atomic.Atomi
* Implementation of a log cleaner that checks if a log is still scheduled for
* replication before deleting it when its TTL is over.
*/
+@Deprecated
public class ReplicationLogCleaner implements LogCleanerDelegate, Watcher {
private static final Log LOG =
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Wed Apr 30 18:18:34 2014
@@ -37,6 +37,7 @@ import java.io.InputStream;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.NetworkInterface;
+import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Enumeration;
@@ -71,7 +72,7 @@ public class HQuorumPeer {
try {
Properties zkProperties = makeZKProps(conf);
writeMyID(zkProperties);
- QuorumPeerConfig zkConfig = new QuorumPeerConfig();
+ HQuorumPeerConfig zkConfig = new HQuorumPeerConfig();
zkConfig.parseProperties(zkProperties);
runZKServer(zkConfig);
} catch (Exception e) {
@@ -93,7 +94,9 @@ public class HQuorumPeer {
}
private static boolean addressIsLocalHost(String address) {
- return address.equals("localhost") || address.equals("127.0.0.1");
+ // localhost address for IPv4 and IPv6 addresses
+ return address.equals("localhost") || address.equals("127.0.0.1") ||
+ address.equals("localhost6") || address.equals("::1");
}
static void writeMyID(Properties properties) throws IOException {
@@ -119,7 +122,12 @@ public class HQuorumPeer {
while(rawAdrs.hasMoreElements()) {
InetAddress inet = (InetAddress) rawAdrs.nextElement();
ips.add(StringUtils.simpleHostname(inet.getHostName()));
- ips.add(inet.getHostAddress());
+ String hostAddr = inet.getHostAddress();
+ ips.add(hostAddr);
+ // Add the global IPv6 address without the scope id part
+ if (hostAddr.contains(":") && hostAddr.contains("%")) {
+ ips.add(hostAddr.substring(0, hostAddr.indexOf("%")));
+ }
}
}
@@ -129,8 +137,24 @@ public class HQuorumPeer {
if (key.startsWith("server.")) {
int dot = key.indexOf('.');
long id = Long.parseLong(key.substring(dot + 1));
- String[] parts = value.split(":");
+ /* According to trunk version of QuorumPeerConfig$parseProperties
+ * Treat an address contained with [ ] as an IPv6 address if it
+ * contains only hex digits and colons. IPv6 addresses will
+ * recognized only if specified in this format.
+ */
+ boolean ipv6 = value.matches("\\[[0-9a-fA-F:]*\\].*");
+ String parts[];
+ if (ipv6) {
+ String blocks[] = value.split("]");
+ String ipv6Address = blocks[0].substring(1);
+ parts = blocks[1].split(":");
+ // The first element in "parts" should be the IP address.
+ parts[0] = ipv6Address;
+ } else {
+ parts = value.split(":");
+ }
String address = parts[0];
+
if (addressIsLocalHost(address) || ips.contains(address)) {
myId = id;
break;
@@ -165,7 +189,7 @@ public class HQuorumPeer {
* @param conf Configuration to read from.
* @return Properties holding mappings representing ZooKeeper zoo.cfg file.
*/
- public static Properties makeZKProps(Configuration conf) {
+ public static Properties makeZKProps(Configuration conf) throws SocketException {
// First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
// it and grab its configuration properties.
ClassLoader cl = HQuorumPeer.class.getClassLoader();
@@ -240,7 +264,25 @@ public class HQuorumPeer {
clientPort = value;
}
else if (key.startsWith("server.")) {
- String host = value.substring(0, value.indexOf(':'));
+ /* According to trunk version of QuorumPeerConfig$parseProperties
+ * Treat an address contained with [ ] as an IPv6 address if it
+ * contains only hex digits and colons. IPv6 addresses will
+ * recognized only if specified in this format.
+ */
+ boolean ipv6 = value.matches("\\[[0-9a-fA-F:]*\\].*");
+ String parts[];
+ if (ipv6) {
+ String blocks[] = value.split("]");
+ String ipv6Address = blocks[0].substring(1);
+ parts = blocks[1].split(":");
+ // The first element in "parts" should be the IP address.
+ // enclose with brackets
+ parts[0] = "[" + ipv6Address + "]";
+ } else {
+ parts = value.split(":");
+ }
+
+ String host = parts[0];
servers.add(host);
try {
//noinspection ResultOfMethodCallIgnored
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeerConfig.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeerConfig.java?rev=1591434&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeerConfig.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeerConfig.java Wed Apr 30 18:18:34 2014
@@ -0,0 +1,441 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType;
+import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
+import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical;
+import org.apache.zookeeper.server.quorum.flexible.QuorumMaj;
+import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
+
+/**
+ * This is a copy of QuorumPeerConfig from ZooKeeper trunk
+ * This file includes the diff https://phabricator.fb.com/D1226868
+ * This is a quick fix to resolve IPv6 issues in the old-stocked ZK
+ * as we are moving toward Zeus.
+ */
+public class HQuorumPeerConfig extends QuorumPeerConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(HQuorumPeerConfig.class);
+ public static final int DYNAMIC_OBSERVER_ID = -1;
+ protected InetSocketAddress clientPortAddress;
+ protected String dataDir;
+ protected String dataLogDir;
+ protected int tickTime = ZooKeeperServer.DEFAULT_TICK_TIME;
+ protected int maxClientCnxns = 60;
+ /** defaults to -1 if not set explicitly */
+ protected int minSessionTimeout = -1;
+ /** defaults to -1 if not set explicitly */
+ protected int maxSessionTimeout = -1;
+ protected boolean localSessionsEnabled = false;
+ protected boolean localSessionsUpgradingEnabled = false;
+
+ protected int initLimit;
+ protected int syncLimit;
+ protected int electionAlg = 3;
+ protected int electionPort = 2182;
+ protected final HashMap<Long,QuorumServer> servers =
+ new HashMap<Long, QuorumServer>();
+ protected final HashMap<Long,QuorumServer> observers =
+ new HashMap<Long, QuorumServer>();
+
+ protected long serverId;
+ protected HashMap<Long, Long> serverWeight = new HashMap<Long, Long>();
+ protected HashMap<Long, Long> serverGroup = new HashMap<Long, Long>();
+ protected int numGroups = 0;
+ protected QuorumVerifier quorumVerifier;
+ protected int snapRetainCount = 3;
+ protected int purgeInterval = 0;
+
+ protected LearnerType peerType = LearnerType.PARTICIPANT;
+
+ protected String ensembleName;
+
+ /**
+ * Minimum snapshot retain count.
+ * @see org.apache.zookeeper.server.PurgeTxnLog#purge(File, File, int)
+ */
+ private final int MIN_SNAP_RETAIN_COUNT = 3;
+
+ /**
+ * Parse a ZooKeeper configuration file
+ * @param path the patch of the configuration file
+ * @throws ConfigException error processing configuration
+ */
+ public void parse(String path) throws ConfigException {
+ File configFile = new File(path);
+
+ LOG.info("Reading configuration from: " + configFile);
+
+ try {
+ if (!configFile.exists()) {
+ throw new IllegalArgumentException(configFile.toString()
+ + " file is missing");
+ }
+
+ Properties cfg = new Properties();
+ FileInputStream in = new FileInputStream(configFile);
+ try {
+ cfg.load(in);
+ } finally {
+ in.close();
+ }
+
+ parseProperties(cfg);
+ } catch (IOException e) {
+ throw new ConfigException("Error processing " + path, e);
+ } catch (IllegalArgumentException e) {
+ throw new ConfigException("Error processing " + path, e);
+ }
+ }
+
+ /**
+ * Parse config from a Properties.
+ * @param zkProp Properties to parse from.
+ * @throws IOException
+ * @throws ConfigException
+ */
+ public void parseProperties(Properties zkProp)
+ throws IOException, ConfigException {
+ int clientPort = 0;
+ String clientPortAddress = null;
+ for (Entry<Object, Object> entry : zkProp.entrySet()) {
+ String key = entry.getKey().toString().trim();
+ String value = entry.getValue().toString().trim();
+ if (key.equals("dataDir")) {
+ dataDir = value;
+ } else if (key.equals("dataLogDir")) {
+ dataLogDir = value;
+ } else if (key.equals("clientPort")) {
+ clientPort = Integer.parseInt(value);
+ } else if (key.equals("localSessionsEnabled")) {
+ localSessionsEnabled = Boolean.parseBoolean(value);
+ } else if (key.equals("localSessionsUpgradingEnabled")) {
+ localSessionsUpgradingEnabled = Boolean.parseBoolean(value);
+ } else if (key.equals("ensembleName")) {
+ ensembleName = value;
+ } else if (key.equals("clientPortAddress")) {
+ clientPortAddress = value.trim();
+ } else if (key.equals("tickTime")) {
+ tickTime = Integer.parseInt(value);
+ } else if (key.equals("maxClientCnxns")) {
+ maxClientCnxns = Integer.parseInt(value);
+ } else if (key.equals("minSessionTimeout")) {
+ minSessionTimeout = Integer.parseInt(value);
+ } else if (key.equals("maxSessionTimeout")) {
+ maxSessionTimeout = Integer.parseInt(value);
+ } else if (key.equals("initLimit")) {
+ initLimit = Integer.parseInt(value);
+ } else if (key.equals("syncLimit")) {
+ syncLimit = Integer.parseInt(value);
+ } else if (key.equals("electionAlg")) {
+ electionAlg = Integer.parseInt(value);
+ } else if (key.equals("peerType")) {
+ if (value.toLowerCase().equals("observer")) {
+ peerType = LearnerType.OBSERVER;
+ } else if (value.toLowerCase().equals("participant")) {
+ peerType = LearnerType.PARTICIPANT;
+ } else
+ {
+ throw new ConfigException("Unrecognised peertype: " + value);
+ }
+ } else if (key.equals("autopurge.snapRetainCount")) {
+ snapRetainCount = Integer.parseInt(value);
+ } else if (key.equals("autopurge.purgeInterval")) {
+ purgeInterval = Integer.parseInt(value);
+ } else if (key.startsWith("server.")) {
+ int dot = key.indexOf('.');
+ long sid = Long.parseLong(key.substring(dot + 1));
+
+ /*
+ * Treat an address contained with [ ] as an IPv6 address if it
+ * contains only hex digits and colons. IPv6 addresses will
+ * recognized only if specified in this format.
+ */
+ boolean ipv6 = value.matches("\\[[0-9a-fA-F:]*\\].*");
+ String parts[];
+ if (ipv6) {
+ String blocks[] = value.split("]");
+ String ipv6Address = blocks[0].substring(1);
+ parts = blocks[1].split(":");
+
+ // The first element in "parts" should be the IP address.
+ parts[0] = ipv6Address;
+ } else {
+ parts = value.split(":");
+ }
+ if ((parts.length != 2) && (parts.length != 3) && (parts.length !=4)) {
+ LOG.error(value
+ + " does not have the form host:port or host:port:port " +
+ " or host:port:port:type");
+ throw new ConfigException("Unrecognised server config: " +
+ value);
+ }
+ InetSocketAddress addr = new InetSocketAddress(parts[0],
+ Integer.parseInt(parts[1]));
+ if (addr.getAddress() == null) {
+ LOG.error("Could not resolve {} into an InetAddress",
+ parts[0]);
+ }
+ if (parts.length == 2) {
+ servers.put(Long.valueOf(sid), new QuorumServer(sid, addr));
+ } else if (parts.length == 3) {
+ InetSocketAddress electionAddr = new InetSocketAddress(
+ parts[0], Integer.parseInt(parts[2]));
+ servers.put(Long.valueOf(sid), new QuorumServer(sid, addr,
+ electionAddr));
+ } else if (parts.length == 4) {
+ InetSocketAddress electionAddr = new InetSocketAddress(
+ parts[0], Integer.parseInt(parts[2]));
+ LearnerType type = LearnerType.PARTICIPANT;
+ if (parts[3].toLowerCase().equals("observer")) {
+ type = LearnerType.OBSERVER;
+ observers.put(Long.valueOf(sid), new QuorumServer(sid, addr,
+ electionAddr,type));
+ } else if (parts[3].toLowerCase().equals("participant")) {
+ type = LearnerType.PARTICIPANT;
+ servers.put(Long.valueOf(sid), new QuorumServer(sid, addr,
+ electionAddr,type));
+ } else {
+ throw new ConfigException("Unrecognised peertype: " + value);
+ }
+ }
+ } else if (key.startsWith("group")) {
+ int dot = key.indexOf('.');
+ long gid = Long.parseLong(key.substring(dot + 1));
+
+ numGroups++;
+
+ String parts[] = value.split(":");
+ for(String s : parts){
+ long sid = Long.parseLong(s);
+ if(serverGroup.containsKey(sid))
+ throw new ConfigException("Server " + sid + "is in multiple groups");
+ else
+ serverGroup.put(sid, gid);
+ }
+
+ } else if(key.startsWith("weight")) {
+ int dot = key.indexOf('.');
+ long sid = Long.parseLong(key.substring(dot + 1));
+ serverWeight.put(sid, Long.parseLong(value));
+ } else {
+ System.setProperty("zookeeper." + key, value);
+ }
+ }
+
+ // Reset to MIN_SNAP_RETAIN_COUNT if invalid (less than 3)
+ // PurgeTxnLog.purge(File, File, int) will not allow to purge less
+ // than 3.
+ if (snapRetainCount < MIN_SNAP_RETAIN_COUNT) {
+ LOG.warn("Invalid autopurge.snapRetainCount: " + snapRetainCount
+ + ". Defaulting to " + MIN_SNAP_RETAIN_COUNT);
+ snapRetainCount = MIN_SNAP_RETAIN_COUNT;
+ }
+
+ if (dataDir == null) {
+ throw new IllegalArgumentException("dataDir is not set");
+ }
+
+ if (dataLogDir == null) {
+ dataLogDir = dataDir;
+ } else {
+ if (!new File(dataLogDir).isDirectory()) {
+ throw new IllegalArgumentException("dataLogDir " + dataLogDir
+ + " is missing.");
+ }
+ }
+ if (clientPort == 0) {
+ throw new IllegalArgumentException("clientPort is not set");
+ }
+ if (clientPortAddress != null) {
+ this.clientPortAddress = new InetSocketAddress(
+ InetAddress.getByName(clientPortAddress), clientPort);
+ } else {
+ this.clientPortAddress = new InetSocketAddress(clientPort);
+ }
+
+ if (tickTime == 0) {
+ throw new IllegalArgumentException("tickTime is not set");
+ }
+ if (minSessionTimeout > maxSessionTimeout) {
+ throw new IllegalArgumentException(
+ "minSessionTimeout must not be larger than maxSessionTimeout");
+ }
+ if (servers.size() == 0) {
+ if (observers.size() > 0) {
+ throw new IllegalArgumentException("Observers w/o participants is an invalid configuration");
+ }
+ // Not a quorum configuration so return immediately - not an error
+ // case (for b/w compatibility), server will default to standalone
+ // mode.
+ return;
+ } else {
+ if (servers.size() <= 2) {
+ LOG.warn("No server failure will be tolerated. " +
+ "You need at least 3 servers.");
+ } else if (servers.size() % 2 == 0) {
+ LOG.warn("Non-optimial configuration, consider an odd number of servers.");
+ }
+ if (initLimit == 0) {
+ throw new IllegalArgumentException("initLimit is not set");
+ }
+ if (syncLimit == 0) {
+ throw new IllegalArgumentException("syncLimit is not set");
+ }
+ /*
+ * If using FLE, then every server requires a separate election
+ * port.
+ */
+ if (electionAlg != 0) {
+ for (QuorumServer s : servers.values()) {
+ if (s.electionAddr == null)
+ throw new IllegalArgumentException(
+ "Missing election port for server: " + s.id);
+ }
+ }
+
+ /*
+ * Default of quorum config is majority
+ */
+ if(serverGroup.size() > 0){
+ if(servers.size() != serverGroup.size())
+ throw new ConfigException("Every server must be in exactly one group");
+ /*
+ * The deafult weight of a server is 1
+ */
+ for(QuorumServer s : servers.values()){
+ if(!serverWeight.containsKey(s.id))
+ serverWeight.put(s.id, (long) 1);
+ }
+
+ /*
+ * Set the quorumVerifier to be QuorumHierarchical
+ */
+ quorumVerifier = new QuorumHierarchical(numGroups,
+ serverWeight, serverGroup);
+ } else {
+ /*
+ * The default QuorumVerifier is QuorumMaj
+ */
+
+ LOG.info("Defaulting to majority quorums");
+ quorumVerifier = new QuorumMaj(servers.size());
+ }
+
+ // Now add observers to servers, once the quorums have been
+ // figured out
+ servers.putAll(observers);
+
+ File myIdFile = new File(dataDir, "myid");
+ if (!myIdFile.exists()) {
+ throw new IllegalArgumentException(myIdFile.toString()
+ + " file is missing");
+ }
+ BufferedReader br = new BufferedReader(new FileReader(myIdFile));
+ String myIdString;
+ try {
+ myIdString = br.readLine();
+ } finally {
+ br.close();
+ }
+ try {
+ serverId = Long.parseLong(myIdString);
+ MDC.put("myid", myIdString);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("serverid " + myIdString
+ + " is not a number");
+ }
+
+ // Warn about inconsistent peer type
+ LearnerType roleByServersList = observers.containsKey(serverId) ? LearnerType.OBSERVER
+ : LearnerType.PARTICIPANT;
+ if (roleByServersList != peerType) {
+ LOG.warn("Peer type from servers list (" + roleByServersList
+ + ") doesn't match peerType (" + peerType
+ + "). Defaulting to servers list.");
+
+ peerType = roleByServersList;
+ }
+ }
+ }
+
+ public InetSocketAddress getClientPortAddress() { return clientPortAddress; }
+ public String getDataDir() { return dataDir; }
+ public String getDataLogDir() { return dataLogDir; }
+ public int getTickTime() { return tickTime; }
+ public int getMaxClientCnxns() { return maxClientCnxns; }
+ public int getMinSessionTimeout() { return minSessionTimeout; }
+ public int getMaxSessionTimeout() { return maxSessionTimeout; }
+ public boolean areLocalSessionsEnabled() { return localSessionsEnabled; }
+ public boolean isLocalSessionsUpgradingEnabled() {
+ return localSessionsUpgradingEnabled;
+ }
+
+ public String getEnsembleName() {
+ return ensembleName;
+ }
+
+ public int getInitLimit() { return initLimit; }
+ public int getSyncLimit() { return syncLimit; }
+ public int getElectionAlg() { return electionAlg; }
+ public int getElectionPort() { return electionPort; }
+
+ public int getSnapRetainCount() {
+ return snapRetainCount;
+ }
+
+ public int getPurgeInterval() {
+ return purgeInterval;
+ }
+
+ public QuorumVerifier getQuorumVerifier() {
+ return quorumVerifier;
+ }
+
+ public Map<Long,QuorumServer> getServers() {
+ return Collections.unmodifiableMap(servers);
+ }
+
+ public long getServerId() { return serverId; }
+
+ public boolean isDistributed() { return servers.size() > 1; }
+
+ public LearnerType getPeerType() {
+ return peerType;
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java Wed Apr 30 18:18:34 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import java.net.SocketException;
import java.util.Map.Entry;
import java.util.Properties;
@@ -36,7 +37,7 @@ public class ZKServerTool {
* Run the tool.
* @param args Command line arguments. First arg is path to zookeepers file.
*/
- public static void main(String args[]) {
+ public static void main(String args[]) throws SocketException {
Configuration conf = HBaseConfiguration.create();
// Note that we do not simply grab the property
// HConstants.ZOOKEEPER_QUORUM from the HBaseConfiguration because the
@@ -46,7 +47,23 @@ public class ZKServerTool {
String key = entry.getKey().toString().trim();
String value = entry.getValue().toString().trim();
if (key.startsWith("server.")) {
- String[] parts = value.split(":");
+ /* According to QuorumPeerConfig$parseProperties
+ * Treat an address contained with [ ] as an IPv6 address if it
+ * contains only hex digits and colons. IPv6 addresses will
+ * recognized only if specified in this format.
+ */
+ boolean ipv6 = value.matches("\\[[0-9a-fA-F:]*\\].*");
+ String parts[];
+ if (ipv6) {
+ String blocks[] = value.split("]");
+ String ipv6Address = blocks[0].substring(1);
+ parts = blocks[1].split(":");
+ // The first element in "parts" should be the IP address.
+ parts[0] = ipv6Address;
+ } else {
+ parts = value.split(":");
+ }
+
String host = parts[0];
System.out.println(host);
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Wed Apr 30 18:18:34 2014
@@ -28,14 +28,18 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
import java.net.Socket;
+import java.net.SocketException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -187,7 +191,8 @@ public class ZooKeeperWrapper implements
}
// return the singleton given the name of the instance
- public static ZooKeeperWrapper getInstance(Configuration conf, String name) {
+ public static ZooKeeperWrapper getInstance(Configuration conf, String name)
+ throws SocketException {
name = getZookeeperClusterKey(conf, name);
return INSTANCES.get(currentNamespaceForTesting + name);
}
@@ -1206,7 +1211,7 @@ public class ZooKeeperWrapper implements
* @param conf Configuration to use to build the key
* @return ensemble key without a name
*/
- public static String getZookeeperClusterKey(Configuration conf) {
+ public static String getZookeeperClusterKey(Configuration conf) throws SocketException {
return getZookeeperClusterKey(conf, null);
}
@@ -1217,7 +1222,8 @@ public class ZooKeeperWrapper implements
* @param name Name that should be appended at the end if not empty or null
* @return ensemble key with a name (if any)
*/
- public static String getZookeeperClusterKey(Configuration conf, String name) {
+ public static String getZookeeperClusterKey(Configuration conf, String name)
+ throws SocketException {
String quorum = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
"[\\t\\n\\x0B\\f\\r]", ""));
StringBuilder builder = new StringBuilder(quorum);
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestIPv6HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestIPv6HServerInfo.java?rev=1591434&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestIPv6HServerInfo.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestIPv6HServerInfo.java Wed Apr 30 18:18:34 2014
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Writables;
+import org.junit.Test;
+
+public class TestIPv6HServerInfo {
+
+ @Test
+ public void testHashCodeAndEquals() {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ HServerInfo hsi2 = new HServerInfo(hsa1, 1L);
+ HServerInfo hsi3 = new HServerInfo(hsa1, 2L);
+ HServerInfo hsi4 = new HServerInfo(hsa1, 1L);
+ HServerAddress hsa2 = new HServerAddress("::1", 1235);
+ HServerInfo hsi5 = new HServerInfo(hsa2, 1L);
+ assertEquals(hsi1.hashCode(), hsi2.hashCode());
+ assertTrue(hsi1.equals(hsi2));
+ assertFalse(hsi1.hashCode() == hsi3.hashCode());
+ assertFalse(hsi1.equals(hsi3));
+ assertEquals(hsi1.hashCode(), hsi4.hashCode());
+ assertTrue(hsi1.equals(hsi4));
+ assertFalse(hsi1.hashCode() == hsi5.hashCode());
+ assertFalse(hsi1.equals(hsi5));
+ }
+
+ @Test
+ public void testHServerInfoHServerInfo() {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ HServerInfo hsi2 = new HServerInfo(hsi1);
+ assertEquals(hsi1, hsi2);
+ }
+
+ @Test
+ public void testGetServerAddress() {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ assertEquals(hsi1.getServerAddress(), hsa1);
+ }
+
+ @Test
+ public void testToString() {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ System.out.println(hsi1.toString());
+ }
+
+ @Test
+ public void testReadFields() throws IOException {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ HServerAddress hsa2 = new HServerAddress("::1", 1235);
+ HServerInfo hsi2 = new HServerInfo(hsa2, 1L);
+ byte [] bytes = Writables.getBytes(hsi1);
+ HServerInfo deserialized =
+ (HServerInfo)Writables.getWritable(bytes, new HServerInfo());
+ assertEquals(hsi1, deserialized);
+ bytes = Writables.getBytes(hsi2);
+ deserialized = (HServerInfo)Writables.getWritable(bytes, new HServerInfo());
+ assertNotSame(hsa1, deserialized);
+ }
+
+ @Test
+ public void testCompareTo() {
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ HServerInfo hsi1 = new HServerInfo(hsa1, 1L);
+ HServerAddress hsa2 = new HServerAddress("::1", 1235);
+ HServerInfo hsi2 = new HServerInfo(hsa2, 1L);
+ assertTrue(hsi1.compareTo(hsi1) == 0);
+ assertTrue(hsi2.compareTo(hsi2) == 0);
+ int compare1 = hsi1.compareTo(hsi2);
+ int compare2 = hsi2.compareTo(hsi1);
+ assertTrue((compare1 > 0)? compare2 < 0: compare2 > 0);
+ }
+
+ @Test
+ public void testHostNameToString() throws IOException {
+
+ // Constructor test
+ HServerAddress hsa1 = new HServerAddress("::1", 1234);
+ assertTrue(hsa1.toString().equals("0:0:0:0:0:0:0:1:1234"));
+
+ // Writable
+ File tempFile = new File("test.out");
+ tempFile.createNewFile();
+ assertTrue(tempFile.exists());
+
+ FileOutputStream fos = new FileOutputStream("test.out");
+
+ DataOutputStream dos = new DataOutputStream(fos);
+ dos.writeUTF("::1");
+ dos.writeInt(1234);
+ dos.flush();
+ dos.close();
+
+ FileInputStream fis = new FileInputStream("test.out");
+ DataInputStream dis = new DataInputStream(fis);
+
+ HServerAddress hsa2 = new HServerAddress();
+ hsa2.readFields(dis);
+
+ assertTrue(hsa2.toString().equals("0:0:0:0:0:0:0:1:1234"));
+ dis.close();
+
+ assertTrue(tempFile.delete());
+
+ }
+
+
+ @Test
+ public void testFromServerName() {
+ String host = "::1";
+ int port = 60020;
+ long startCode = 1343258056696L;
+ String serverName = host + HServerInfo.SERVERNAME_SEPARATOR + port +
+ HServerInfo.SERVERNAME_SEPARATOR + startCode;
+ HServerInfo hsi = HServerInfo.fromServerName(serverName);
+ assertEquals(host, hsi.getHostname());
+ assertEquals(port, hsi.getServerAddress().getPort());
+ assertEquals(startCode, hsi.getStartCode());
+ assertTrue(HServerInfo.isValidServerName(serverName));
+ assertEquals(serverName, hsi.getServerName());
+ }
+
+ @Test
+ public void testIsValidServerName() {
+ assertTrue(HServerInfo.isValidServerName("foo.bar,60020," + Long.MAX_VALUE));
+ assertTrue(HServerInfo.isValidServerName("::1,60020," + Long.MAX_VALUE));
+ assertTrue(HServerInfo.isValidServerName("2620::1cfe:b:7ec3:a1ff:fe86:5c5e,60020," + Long.MAX_VALUE));
+ assertTrue(HServerInfo.isValidServerName("2401:db00:2030:507b:face:0:17:0,60021,1396986476081"));
+ assertTrue(HServerInfo.isValidServerName("fe80::6203:8ff:fe89:8590%en0,60020," + Long.MAX_VALUE));
+ assertTrue(HServerInfo.isValidServerName("www.acme.com,80," + Long.MIN_VALUE));
+ assertFalse(HServerInfo.isValidServerName(",www.acme.com,80,0"));
+ assertTrue(HServerInfo.isValidServerName("foo.bar,60020," + Long.MAX_VALUE));
+ assertFalse(HServerInfo.isValidServerName("foo.bar,60020," + Long.MAX_VALUE + "a"));
+ assertFalse(HServerInfo.isValidServerName(",60020," + Long.MAX_VALUE));
+ assertFalse(HServerInfo.isValidServerName(" ,60020," + Long.MAX_VALUE));
+ assertFalse(HServerInfo.isValidServerName(" ,60020," + Long.MAX_VALUE));
+ assertFalse(HServerInfo.isValidServerName("!>;@#@#localhost,60020," +
+ Long.MAX_VALUE));
+ }
+
+}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java Wed Apr 30 18:18:34 2014
@@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
@@ -68,7 +69,7 @@ public class TestRegionStateOnMasterFail
private static void assertHostPort(String rsName) {
assertTrue(rsName + " is not a valid host:port pair",
- ServerManager.HOST_PORT_RE.matcher(rsName).matches());
+ HServerInfo.isValidHostAndPort(rsName));
}
private static final byte[] TABLE_NAME = Bytes.toBytes("TestRegionState");
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java?rev=1591434&r1=1591433&r2=1591434&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java Wed Apr 30 18:18:34 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.zookeepe
import java.io.ByteArrayInputStream;
import java.io.InputStream;
+import java.net.SocketException;
import java.util.Map;
import java.util.Properties;
@@ -29,7 +30,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
/**
@@ -63,7 +63,7 @@ public class TestHQuorumPeer extends HBa
}
/** */
- public void testMakeZKProps() {
+ public void testMakeZKProps() throws SocketException {
Properties properties = HQuorumPeer.makeZKProps(conf);
assertEquals(dataDir.toString(), properties.get("dataDir"));
assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
@@ -71,14 +71,16 @@ public class TestHQuorumPeer extends HBa
assertEquals(null, properties.get("server.1"));
String oldValue = conf.get(HConstants.ZOOKEEPER_QUORUM);
- conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar");
+ conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar," +
+ "[2620::1cfe:b:7ec3:a1ff:fe86:5c5e]");
properties = HQuorumPeer.makeZKProps(conf);
assertEquals(dataDir.toString(), properties.get("dataDir"));
assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
assertEquals("a.foo.bar:2888:3888", properties.get("server.0"));
assertEquals("b.foo.bar:2888:3888", properties.get("server.1"));
assertEquals("c.foo.bar:2888:3888", properties.get("server.2"));
- assertEquals(null, properties.get("server.3"));
+ assertEquals("[2620::1cfe:b:7ec3:a1ff:fe86:5c5e]:2888:3888", properties.get("server.3"));
+ assertEquals(null, properties.get("server.4"));
conf.set(HConstants.ZOOKEEPER_QUORUM, oldValue);
}
@@ -91,7 +93,8 @@ public class TestHQuorumPeer extends HBa
"syncLimit=2\n" +
"server.0=${hbase.master.hostname}:2888:3888\n" +
"server.1=server1:2888:3888\n" +
- "server.2=server2:2888:3888\n";
+ "server.2=server2:2888:3888\n" +
+ "server.3=[2620::1cfe:b:7ec3:a1ff:fe86:5c5e]:2888:3888\n";
System.setProperty("hbase.master.hostname", "localhost");
InputStream is = new ByteArrayInputStream(s.getBytes());
@@ -104,13 +107,13 @@ public class TestHQuorumPeer extends HBa
assertEquals("localhost:2888:3888", properties.get("server.0"));
HQuorumPeer.writeMyID(properties);
- QuorumPeerConfig config = new QuorumPeerConfig();
+ HQuorumPeerConfig config = new HQuorumPeerConfig();
config.parseProperties(properties);
assertEquals(dataDir.toString(), config.getDataDir());
assertEquals(2181, config.getClientPortAddress().getPort());
Map<Long,QuorumServer> servers = config.getServers();
- assertEquals(3, servers.size());
+ assertEquals(4, servers.size());
assertTrue(servers.containsKey(Long.valueOf(0)));
QuorumServer server = servers.get(Long.valueOf(0));
assertEquals("localhost", server.addr.getHostName());
@@ -131,7 +134,7 @@ public class TestHQuorumPeer extends HBa
/**
* Test Case for HBASE-2305
*/
- public void testShouldAssignDefaultZookeeperClientPort() {
+ public void testShouldAssignDefaultZookeeperClientPort() throws SocketException {
Configuration config = HBaseConfiguration.create();
config.clear();
Properties p = HQuorumPeer.makeZKProps(config);