You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/05/02 18:26:38 UTC
svn commit: r1333099 [4/5] - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/executor/
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/protobuf/
main/java/org/apache/hadoop/hbase/pro...
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java Wed May 2 16:26:36 2012
@@ -19,17 +19,13 @@ package org.apache.hadoop.hbase.zookeepe
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RootRegionServer;
-import org.apache.hadoop.hbase.util.Addressing;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.zookeeper.KeeperException;
-import com.google.protobuf.InvalidProtocolBufferException;
-
/**
* Tracks the root region server location node in zookeeper.
* Root region location is set by {@link RootLocationEditor} usually called
@@ -64,7 +60,12 @@ public class RootRegionTracker extends Z
* @throws InterruptedException
*/
public ServerName getRootRegionLocation() throws InterruptedException {
- return ZKUtil.znodeContentToServerName(super.getData(true));
+ try {
+ return ServerName.parseFrom(super.getData(true));
+ } catch (DeserializationException e) {
+ LOG.warn("Failed parse", e);
+ return null;
+ }
}
/**
@@ -76,7 +77,11 @@ public class RootRegionTracker extends Z
*/
public static ServerName getRootRegionLocation(final ZooKeeperWatcher zkw)
throws KeeperException {
- return ZKUtil.znodeContentToServerName(ZKUtil.getData(zkw, zkw.rootServerZNode));
+ try {
+ return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.rootServerZNode));
+ } catch (DeserializationException e) {
+ throw ZKUtil.convert(e);
+ }
}
/**
@@ -97,7 +102,12 @@ public class RootRegionTracker extends Z
LOG.error(errorMsg);
throw new IllegalArgumentException(errorMsg);
}
- return ZKUtil.znodeContentToServerName(super.blockUntilAvailable(timeout, true));
+ try {
+ return ServerName.parseFrom(super.blockUntilAvailable(timeout, true));
+ } catch (DeserializationException e) {
+ LOG.warn("Failed parse", e);
+ return null;
+ }
}
/**
@@ -113,7 +123,7 @@ public class RootRegionTracker extends Z
LOG.info("Setting ROOT region location in ZooKeeper as " + location);
// Make the RootRegionServer pb and then get its bytes and save this as
// the znode content.
- byte [] data = getZNodeData(location);
+ byte [] data = toByteArray(location);
try {
ZKUtil.createAndWatch(zookeeper, zookeeper.rootServerZNode, data);
} catch(KeeperException.NodeExistsException nee) {
@@ -127,7 +137,7 @@ public class RootRegionTracker extends Z
* @param sn What to put into the znode.
* @return The content of the root-region-server znode
*/
- static byte [] getZNodeData(final ServerName sn) {
+ static byte [] toByteArray(final ServerName sn) {
// ZNode content is a pb message preceeded by some pb magic.
HBaseProtos.ServerName pbsn =
HBaseProtos.ServerName.newBuilder().setHostName(sn.getHostname()).
@@ -164,6 +174,12 @@ public class RootRegionTracker extends Z
final long timeout)
throws InterruptedException {
byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.rootServerZNode, timeout);
- return ZKUtil.znodeContentToServerName(data);
+ if (data == null) return null;
+ try {
+ return ServerName.parseFrom(data);
+ } catch (DeserializationException e) {
+ LOG.warn("Failed parse", e);
+ return null;
+ }
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java Wed May 2 16:26:36 2012
@@ -25,9 +25,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.DeserializationException;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
+// We should not be importing this Type here, nor a RegionTransition, etc. This class should be
+// about zk and bytes only.
import org.apache.hadoop.hbase.executor.EventHandler.EventType;
import org.apache.zookeeper.AsyncCallback;
import org.apache.zookeeper.KeeperException;
@@ -145,10 +149,10 @@ public class ZKAssign {
throws KeeperException, KeeperException.NodeExistsException {
LOG.debug(zkw.prefix("Creating unassigned node for " +
region.getEncodedName() + " in OFFLINE state"));
- RegionTransitionData data = new RegionTransitionData(event,
- region.getRegionName(), serverName);
+ RegionTransition rt =
+ RegionTransition.createRegionTransition(event, region.getRegionName(), serverName);
String node = getNodeName(zkw, region.getEncodedName());
- ZKUtil.createAndWatch(zkw, node, data.getBytes());
+ ZKUtil.createAndWatch(zkw, node, rt.toByteArray());
}
/**
@@ -172,10 +176,10 @@ public class ZKAssign {
throws KeeperException {
LOG.debug(zkw.prefix("Async create of unassigned node for " +
region.getEncodedName() + " with OFFLINE state"));
- RegionTransitionData data = new RegionTransitionData(
- EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
+ RegionTransition rt =
+ RegionTransition.createRegionTransition(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
String node = getNodeName(zkw, region.getEncodedName());
- ZKUtil.asyncCreate(zkw, node, data.getBytes(), cb, ctx);
+ ZKUtil.asyncCreate(zkw, node, rt.toByteArray(), cb, ctx);
}
/**
@@ -201,10 +205,10 @@ public class ZKAssign {
throws KeeperException, KeeperException.NoNodeException {
LOG.debug(zkw.prefix("Forcing existing unassigned node for " +
region.getEncodedName() + " to OFFLINE state"));
- RegionTransitionData data = new RegionTransitionData(
- EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
+ RegionTransition rt =
+ RegionTransition.createRegionTransition(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
String node = getNodeName(zkw, region.getEncodedName());
- ZKUtil.setData(zkw, node, data.getBytes());
+ ZKUtil.setData(zkw, node, rt.toByteArray());
}
/**
@@ -267,8 +271,9 @@ public class ZKAssign {
throws KeeperException {
LOG.debug(zkw.prefix("Creating (or updating) unassigned node for " +
region.getEncodedName() + " with OFFLINE state"));
- RegionTransitionData data = new RegionTransitionData(
- EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
+ RegionTransition rt = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_OFFLINE,
+ region.getRegionName(), serverName, HConstants.EMPTY_BYTE_ARRAY);
+ byte [] data = rt.toByteArray();
String node = getNodeName(zkw, region.getEncodedName());
Stat stat = new Stat();
zkw.sync(node);
@@ -282,15 +287,15 @@ public class ZKAssign {
if (hijack && !allowCreation) {
return -1;
}
- return ZKUtil.createAndWatch(zkw, node, data.getBytes());
+ return ZKUtil.createAndWatch(zkw, node, data);
} else {
- RegionTransitionData curDataInZNode = ZKAssign.getDataNoWatch(zkw, region
- .getEncodedName(), stat);
+ byte [] curDataInZNode = ZKAssign.getDataNoWatch(zkw, region.getEncodedName(), stat);
+ RegionTransition curRt = getRegionTransition(curDataInZNode);
// Do not move the node to OFFLINE if znode is in any of the following
// state.
// Because these are already executed states.
- if (hijack && null != curDataInZNode) {
- EventType eventType = curDataInZNode.getEventType();
+ if (hijack && curRt != null) {
+ EventType eventType = curRt.getEventType();
if (eventType.equals(EventType.M_ZK_REGION_CLOSING)
|| eventType.equals(EventType.RS_ZK_REGION_CLOSED)
|| eventType.equals(EventType.RS_ZK_REGION_OPENED)) {
@@ -300,7 +305,7 @@ public class ZKAssign {
boolean setData = false;
try {
- setData = ZKUtil.setData(zkw, node, data.getBytes(), version);
+ setData = ZKUtil.setData(zkw, node, data, version);
// Setdata throws KeeperException which aborts the Master. So we are
// catching it here.
// If just before setting the znode to OFFLINE if the RS has made any
@@ -315,9 +320,9 @@ public class ZKAssign {
} else {
// We successfully forced to OFFLINE, reset watch and handle if
// the state changed in between our set and the watch
- RegionTransitionData curData =
- ZKAssign.getData(zkw, region.getEncodedName());
- if (curData.getEventType() != data.getEventType()) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ rt = getRegionTransition(bytes);
+ if (rt.getEventType() != EventType.M_ZK_REGION_OFFLINE) {
// state changed, need to process
return -1;
}
@@ -494,11 +499,11 @@ public class ZKAssign {
// If it came back null, node does not exist.
throw KeeperException.create(Code.NONODE);
}
- RegionTransitionData data = RegionTransitionData.fromBytes(bytes);
- if (!data.getEventType().equals(expectedState)) {
- LOG.warn(zkw.prefix("Attempting to delete unassigned " +
- "node " + regionName + " in " + expectedState +
- " state but node is in " + data.getEventType() + " state"));
+ RegionTransition rt = getRegionTransition(bytes);
+ EventType et = rt.getEventType();
+ if (!et.equals(expectedState)) {
+ LOG.warn(zkw.prefix("Attempting to delete unassigned node " + regionName + " in " +
+ expectedState + " state but node is in " + et + " state"));
return false;
}
if (expectedVersion != -1
@@ -564,12 +569,10 @@ public class ZKAssign {
throws KeeperException, KeeperException.NodeExistsException {
LOG.debug(zkw.prefix("Creating unassigned node for " +
region.getEncodedName() + " in a CLOSING state"));
-
- RegionTransitionData data = new RegionTransitionData(
- EventType.M_ZK_REGION_CLOSING, region.getRegionName(), serverName);
-
+ RegionTransition rt = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
+ region.getRegionName(), serverName, HConstants.EMPTY_BYTE_ARRAY);
String node = getNodeName(zkw, region.getEncodedName());
- return ZKUtil.createAndWatch(zkw, node, data.getBytes());
+ return ZKUtil.createAndWatch(zkw, node, rt.toByteArray());
}
/**
@@ -748,8 +751,7 @@ public class ZKAssign {
ServerName serverName, EventType beginState, EventType endState,
int expectedVersion)
throws KeeperException {
- return transitionNode(zkw, region, serverName, beginState, endState,
- expectedVersion, null);
+ return transitionNode(zkw, region, serverName, beginState, endState, expectedVersion, null);
}
public static int transitionNode(ZooKeeperWatcher zkw, HRegionInfo region,
@@ -773,11 +775,10 @@ public class ZKAssign {
// Node no longer exists. Return -1. It means unsuccessful transition.
return -1;
}
- RegionTransitionData existingData =
- RegionTransitionData.fromBytes(existingBytes);
+ RegionTransition rt = getRegionTransition(existingBytes);
// Verify it is the expected version
- if(expectedVersion != -1 && stat.getVersion() != expectedVersion) {
+ if (expectedVersion != -1 && stat.getVersion() != expectedVersion) {
LOG.warn(zkw.prefix("Attempt to transition the " +
"unassigned node for " + encoded +
" from " + beginState + " to " + endState + " failed, " +
@@ -799,20 +800,19 @@ public class ZKAssign {
}
// Verify it is in expected state
- if(!existingData.getEventType().equals(beginState)) {
+ EventType et = rt.getEventType();
+ if (!et.equals(beginState)) {
LOG.warn(zkw.prefix("Attempt to transition the " +
"unassigned node for " + encoded +
" from " + beginState + " to " + endState + " failed, " +
- "the node existed but was in the state " + existingData.getEventType() +
- " set by the server " + serverName));
+ "the node existed but was in the state " + et + " set by the server " + serverName));
return -1;
}
// Write new data, ensuring data has not changed since we last read it
try {
- RegionTransitionData data = new RegionTransitionData(endState,
- region.getRegionName(), serverName, payload);
- if(!ZKUtil.setData(zkw, node, data.getBytes(), stat.getVersion())) {
+ rt = RegionTransition.createRegionTransition(endState, region.getRegionName(), serverName, payload);
+ if(!ZKUtil.setData(zkw, node, rt.toByteArray(), stat.getVersion())) {
LOG.warn(zkw.prefix("Attempt to transition the " +
"unassigned node for " + encoded +
" from " + beginState + " to " + endState + " failed, " +
@@ -835,6 +835,15 @@ public class ZKAssign {
}
}
+ private static RegionTransition getRegionTransition(final byte [] bytes) throws KeeperException {
+ try {
+ return RegionTransition.parseFrom(bytes);
+ } catch (DeserializationException e) {
+ // Convert to a zk exception for now. Otherwise have to change API
+ throw ZKUtil.convert(e);
+ }
+ }
+
/**
* Gets the current data in the unassigned node for the specified region name
* or fully-qualified path.
@@ -845,19 +854,14 @@ public class ZKAssign {
*
* @param zkw zk reference
* @param pathOrRegionName fully-specified path or region name
- * @return data for the unassigned node
+ * @return znode content
* @throws KeeperException if unexpected zookeeper exception
*/
- public static RegionTransitionData getData(ZooKeeperWatcher zkw,
+ public static byte [] getData(ZooKeeperWatcher zkw,
String pathOrRegionName)
throws KeeperException {
- String node = pathOrRegionName.startsWith("/") ?
- pathOrRegionName : getNodeName(zkw, pathOrRegionName);
- byte [] data = ZKUtil.getDataAndWatch(zkw, node);
- if(data == null) {
- return null;
- }
- return RegionTransitionData.fromBytes(data);
+ String node = getPath(zkw, pathOrRegionName);
+ return ZKUtil.getDataAndWatch(zkw, node);
}
/**
@@ -871,19 +875,14 @@ public class ZKAssign {
* @param zkw zk reference
* @param pathOrRegionName fully-specified path or region name
* @param stat object to populate the version.
- * @return data for the unassigned node
+ * @return znode content
* @throws KeeperException if unexpected zookeeper exception
*/
- public static RegionTransitionData getDataAndWatch(ZooKeeperWatcher zkw,
+ public static byte [] getDataAndWatch(ZooKeeperWatcher zkw,
String pathOrRegionName, Stat stat)
throws KeeperException {
- String node = pathOrRegionName.startsWith("/") ?
- pathOrRegionName : getNodeName(zkw, pathOrRegionName);
- byte [] data = ZKUtil.getDataAndWatch(zkw, node, stat);
- if(data == null) {
- return null;
- }
- return RegionTransitionData.fromBytes(data);
+ String node = getPath(zkw, pathOrRegionName);
+ return ZKUtil.getDataAndWatch(zkw, node, stat);
}
/**
@@ -897,19 +896,23 @@ public class ZKAssign {
* @param zkw zk reference
* @param pathOrRegionName fully-specified path or region name
* @param stat object to store node info into on getData call
- * @return data for the unassigned node or null if node does not exist
+ * @return znode content
* @throws KeeperException if unexpected zookeeper exception
*/
- public static RegionTransitionData getDataNoWatch(ZooKeeperWatcher zkw,
+ public static byte [] getDataNoWatch(ZooKeeperWatcher zkw,
String pathOrRegionName, Stat stat)
throws KeeperException {
- String node = pathOrRegionName.startsWith("/") ?
- pathOrRegionName : getNodeName(zkw, pathOrRegionName);
- byte [] data = ZKUtil.getDataNoWatch(zkw, node, stat);
- if (data == null) {
- return null;
- }
- return RegionTransitionData.fromBytes(data);
+ String node = getPath(zkw, pathOrRegionName);
+ return ZKUtil.getDataNoWatch(zkw, node, stat);
+ }
+
+ /**
+ * @param zkw
+ * @param pathOrRegionName
+ * @return Path to znode
+ */
+ public static String getPath(final ZooKeeperWatcher zkw, final String pathOrRegionName) {
+ return pathOrRegionName.startsWith("/")? pathOrRegionName : getNodeName(zkw, pathOrRegionName);
}
/**
@@ -983,42 +986,18 @@ public class ZKAssign {
}
/**
- * Verifies that the specified region is in the specified state in ZooKeeper.
- * <p>
- * Returns true if region is in transition and in the specified state in
- * ZooKeeper. Returns false if the region does not exist in ZK or is in
- * a different state.
- * <p>
- * Method synchronizes() with ZK so will yield an up-to-date result but is
- * a slow read.
- * @param zkw
- * @param region
- * @param expectedState
- * @return true if region exists and is in expected state
- */
- public static boolean verifyRegionState(ZooKeeperWatcher zkw,
- HRegionInfo region, EventType expectedState)
- throws KeeperException {
- String encoded = region.getEncodedName();
-
- String node = getNodeName(zkw, encoded);
- zkw.sync(node);
-
- // Read existing data of the node
- byte [] existingBytes = null;
+ * Presume bytes are serialized unassigned data structure
+ * @param znodeBytes
+ * @return String of the deserialized znode bytes.
+ */
+ static String toString(final byte[] znodeBytes) {
+ // This method should not exist. Used by ZKUtil stringifying RegionTransition. Have the
+ // method in here so RegionTransition does not leak into ZKUtil.
try {
- existingBytes = ZKUtil.getDataAndWatch(zkw, node);
- } catch (KeeperException.NoNodeException nne) {
- return false;
- } catch (KeeperException e) {
- throw e;
- }
- if (existingBytes == null) return false;
- RegionTransitionData existingData =
- RegionTransitionData.fromBytes(existingBytes);
- if (existingData.getEventType() == expectedState){
- return true;
+ RegionTransition rt = RegionTransition.parseFrom(znodeBytes);
+ return rt.toString();
+ } catch (DeserializationException e) {
+ return "";
}
- return false;
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java Wed May 2 16:26:36 2012
@@ -1,7 +1,5 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
+ * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
@@ -21,13 +19,10 @@ package org.apache.hadoop.hbase.zookeepe
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Field;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,30 +32,23 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.master.SplitLogManager;
import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
-import org.apache.hadoop.hbase.util.Bytes;
/**
- * Common methods and attributes used by {@link SplitLogManager} and
- * {@link SplitLogWorker}
+ * Common methods and attributes used by {@link SplitLogManager} and {@link SplitLogWorker}
+ * running distributed splitting of WAL logs.
*/
@InterfaceAudience.Private
public class ZKSplitLog {
private static final Log LOG = LogFactory.getLog(ZKSplitLog.class);
- public static final int DEFAULT_TIMEOUT = 25000; // 25 sec
- public static final int DEFAULT_ZK_RETRIES = 3;
- public static final int DEFAULT_MAX_RESUBMIT = 3;
- public static final int DEFAULT_UNASSIGNED_TIMEOUT = (3 * 60 * 1000); //3 min
-
/**
* Gets the full path node name for the log file being split.
* This method will url encode the filename.
* @param zkw zk reference
* @param filename log file name (only the basename)
*/
- public static String getEncodedNodeName(ZooKeeperWatcher zkw,
- String filename) {
- return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename));
+ public static String getEncodedNodeName(ZooKeeperWatcher zkw, String filename) {
+ return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename));
}
public static String getFileName(String node) {
@@ -68,8 +56,7 @@ public class ZKSplitLog {
return decode(basename);
}
-
- public static String encode(String s) {
+ static String encode(String s) {
try {
return URLEncoder.encode(s, "UTF-8");
} catch (UnsupportedEncodingException e) {
@@ -77,7 +64,7 @@ public class ZKSplitLog {
}
}
- public static String decode(String s) {
+ static String decode(String s) {
try {
return URLDecoder.decode(s, "UTF-8");
} catch (UnsupportedEncodingException e) {
@@ -107,53 +94,6 @@ public class ZKSplitLog {
return dirname.equals(zkw.splitLogZNode);
}
- public static enum TaskState {
- TASK_UNASSIGNED("unassigned"),
- TASK_OWNED("owned"),
- TASK_RESIGNED("resigned"),
- TASK_DONE("done"),
- TASK_ERR("err");
-
- private final byte[] state;
- private TaskState(String s) {
- state = s.getBytes();
- }
-
- public byte[] get(String serverName) {
- return (Bytes.add(state, " ".getBytes(), serverName.getBytes()));
- }
-
- public String getWriterName(byte[] data) {
- String str = Bytes.toString(data);
- return str.substring(str.indexOf(' ') + 1);
- }
-
-
- /**
- * @param s
- * @return True if {@link #state} is a prefix of s. False otherwise.
- */
- public boolean equals(byte[] s) {
- if (s.length < state.length) {
- return (false);
- }
- for (int i = 0; i < state.length; i++) {
- if (state[i] != s[i]) {
- return (false);
- }
- }
- return (true);
- }
-
- public boolean equals(byte[] s, String serverName) {
- return (Arrays.equals(s, get(serverName)));
- }
- @Override
- public String toString() {
- return new String(state);
- }
- }
-
public static Path getSplitLogDir(Path rootdir, String tmpname) {
return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname);
}
@@ -172,8 +112,8 @@ public class ZKSplitLog {
return ret;
}
- public static String getSplitLogDirTmpComponent(String worker, String file) {
- return (worker + "_" + ZKSplitLog.encode(file));
+ public static String getSplitLogDirTmpComponent(final String worker, String file) {
+ return worker + "_" + ZKSplitLog.encode(file);
}
public static void markCorrupted(Path rootdir, String tmpname,
@@ -198,81 +138,4 @@ public class ZKSplitLog {
public static boolean isCorruptFlagFile(Path file) {
return file.getName().equals("corrupt");
}
-
-
- public static class Counters {
- //SplitLogManager counters
- public static AtomicLong tot_mgr_log_split_batch_start = new AtomicLong(0);
- public static AtomicLong tot_mgr_log_split_batch_success =
- new AtomicLong(0);
- public static AtomicLong tot_mgr_log_split_batch_err = new AtomicLong(0);
- public static AtomicLong tot_mgr_new_unexpected_hlogs = new AtomicLong(0);
- public static AtomicLong tot_mgr_log_split_start = new AtomicLong(0);
- public static AtomicLong tot_mgr_log_split_success = new AtomicLong(0);
- public static AtomicLong tot_mgr_log_split_err = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_create_queued = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_create_result = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_already_exists = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_create_err = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_create_retry = new AtomicLong(0);
- public static AtomicLong tot_mgr_get_data_queued = new AtomicLong(0);
- public static AtomicLong tot_mgr_get_data_result = new AtomicLong(0);
- public static AtomicLong tot_mgr_get_data_nonode = new AtomicLong(0);
- public static AtomicLong tot_mgr_get_data_err = new AtomicLong(0);
- public static AtomicLong tot_mgr_get_data_retry = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_delete_queued = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_delete_result = new AtomicLong(0);
- public static AtomicLong tot_mgr_node_delete_err = new AtomicLong(0);
- public static AtomicLong tot_mgr_resubmit = new AtomicLong(0);
- public static AtomicLong tot_mgr_resubmit_failed = new AtomicLong(0);
- public static AtomicLong tot_mgr_null_data = new AtomicLong(0);
- public static AtomicLong tot_mgr_orphan_task_acquired = new AtomicLong(0);
- public static AtomicLong tot_mgr_wait_for_zk_delete = new AtomicLong(0);
- public static AtomicLong tot_mgr_unacquired_orphan_done = new AtomicLong(0);
- public static AtomicLong tot_mgr_resubmit_threshold_reached =
- new AtomicLong(0);
- public static AtomicLong tot_mgr_missing_state_in_delete =
- new AtomicLong(0);
- public static AtomicLong tot_mgr_heartbeat = new AtomicLong(0);
- public static AtomicLong tot_mgr_rescan = new AtomicLong(0);
- public static AtomicLong tot_mgr_rescan_deleted = new AtomicLong(0);
- public static AtomicLong tot_mgr_task_deleted = new AtomicLong(0);
- public static AtomicLong tot_mgr_resubmit_unassigned = new AtomicLong(0);
- public static AtomicLong tot_mgr_relist_logdir = new AtomicLong(0);
- public static AtomicLong tot_mgr_resubmit_dead_server_task =
- new AtomicLong(0);
-
-
-
- // SplitLogWorker counters
- public static AtomicLong tot_wkr_failed_to_grab_task_no_data =
- new AtomicLong(0);
- public static AtomicLong tot_wkr_failed_to_grab_task_exception =
- new AtomicLong(0);
- public static AtomicLong tot_wkr_failed_to_grab_task_owned =
- new AtomicLong(0);
- public static AtomicLong tot_wkr_failed_to_grab_task_lost_race =
- new AtomicLong(0);
- public static AtomicLong tot_wkr_task_acquired = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_resigned = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_done = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_err = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_heartbeat = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_acquired_rescan = new AtomicLong(0);
- public static AtomicLong tot_wkr_get_data_queued = new AtomicLong(0);
- public static AtomicLong tot_wkr_get_data_result = new AtomicLong(0);
- public static AtomicLong tot_wkr_get_data_retry = new AtomicLong(0);
- public static AtomicLong tot_wkr_preempt_task = new AtomicLong(0);
- public static AtomicLong tot_wkr_task_heartbeat_failed = new AtomicLong(0);
- public static AtomicLong tot_wkr_final_transistion_failed =
- new AtomicLong(0);
-
- public static void resetCounters() throws Exception {
- Class<?> cl = (new Counters()).getClass();
- Field[] flds = cl.getDeclaredFields();
- for (Field fld : flds) {
- ((AtomicLong)fld.get(null)).set(0);
- }
- }
- }
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java Wed May 2 16:26:36 2012
@@ -26,7 +26,6 @@ import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -36,28 +35,21 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.EmptyWatcher;
+import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RootRegionServer;
-import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.zookeeper.AsyncCallback;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
-import com.google.protobuf.InvalidProtocolBufferException;
-
/**
* Internal HBase utility class for ZooKeeper.
*
@@ -359,8 +351,7 @@ public class ZKUtil {
* null if parent does not exist
* @throws KeeperException if unexpected zookeeper exception
*/
- public static List<String> listChildrenNoWatch(
- ZooKeeperWatcher zkw, String znode)
+ public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
throws KeeperException {
List<String> children = null;
try {
@@ -376,7 +367,9 @@ public class ZKUtil {
/**
* Simple class to hold a node path and node data.
+ * @deprecated Unused
*/
+ @Deprecated
public static class NodeAndData {
private String node;
private byte [] data;
@@ -392,7 +385,7 @@ public class ZKUtil {
}
@Override
public String toString() {
- return node + " (" + RegionTransitionData.fromBytes(data) + ")";
+ return node;
}
public boolean isEmpty() {
return (data.length == 0);
@@ -600,6 +593,7 @@ public class ZKUtil {
* @return list of data of children of the specified node, an empty list if the node
* exists but has no children, and null if the node does not exist
* @throws KeeperException if unexpected zookeeper exception
+ * @deprecated Unused
*/
public static List<NodeAndData> getChildDataAndWatchForNewChildren(
ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
@@ -630,6 +624,7 @@ public class ZKUtil {
* @param expectedVersion
* @throws KeeperException if unexpected zookeeper exception
* @throws KeeperException.BadVersionException if version mismatch
+ * @deprecated Unused
*/
public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
byte [] data, int expectedVersion)
@@ -1144,9 +1139,21 @@ public class ZKUtil {
" byte(s) of data from znode " + znode +
(watcherSet? " and set watcher; ": "; data=") +
(data == null? "null": data.length == 0? "empty": (
- znode.startsWith(zkw.assignmentZNode) ?
- RegionTransitionData.fromBytes(data).toString()
- : StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
+ znode.startsWith(zkw.assignmentZNode)?
+ ZKAssign.toString(data): // We should not be doing this reaching into another class
+ znode.startsWith(zkw.rootServerZNode)?
+ getServerNameOrEmptyString(data):
+ znode.startsWith(zkw.backupMasterAddressesZNode)?
+ getServerNameOrEmptyString(data):
+ StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
+ }
+
+ private static String getServerNameOrEmptyString(final byte [] data) {
+ try {
+ return ServerName.parseFrom(data).toString();
+ } catch (DeserializationException e) {
+ return "";
+ }
}
/**
@@ -1222,44 +1229,14 @@ public class ZKUtil {
/**
- * Get a ServerName from the passed in znode data bytes.
- * @param data ZNode data with a server name in it; can handle the old style
- * servername where servername was host and port. Works too with data that
- * begins w/ the pb 'PBUF' magic and that its then followed by a protobuf that
- * has a serialized {@link ServerName} in it.
- * @return Returns null if <code>data</code> is null else converts passed data
- * to a ServerName instance.
- */
- public static ServerName znodeContentToServerName(final byte [] data) {
- if (data == null || data.length <= 0) return null;
- if (ProtobufUtil.isPBMagicPrefix(data)) {
- int prefixLen = ProtobufUtil.lengthOfPBMagic();
- try {
- RootRegionServer rss =
- RootRegionServer.newBuilder().mergeFrom(data, prefixLen, data.length - prefixLen).build();
- HBaseProtos.ServerName sn = rss.getServer();
- return new ServerName(sn.getHostName(), sn.getPort(), sn.getStartCode());
- } catch (InvalidProtocolBufferException e) {
- // A failed parse of the znode is pretty catastrophic. Rather than loop
- // retrying hoping the bad bytes will changes, and rather than change
- // the signature on this method to add an IOE which will send ripples all
- // over the code base, throw a RuntimeException. This should "never" happen.
- // Fail fast if it does.
- throw new RuntimeException(e);
- }
- }
- // The str returned could be old style -- pre hbase-1502 -- which was
- // hostname and port seperated by a colon rather than hostname, port and
- // startcode delimited by a ','.
- String str = Bytes.toString(data);
- int index = str.indexOf(ServerName.SERVERNAME_SEPARATOR);
- if (index != -1) {
- // Presume its ServerName serialized with versioned bytes.
- return ServerName.parseVersionedServerName(data);
- }
- // Presume it a hostname:port format.
- String hostname = Addressing.parseHostname(str);
- int port = Addressing.parsePort(str);
- return new ServerName(hostname, port, -1L);
+ * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
+ * Used when can't let a {@link DeserializationException} out w/o changing public API.
+ * @param e Exception to convert
+ * @return Converted exception
+ */
+ public static KeeperException convert(final DeserializationException e) {
+ KeeperException ke = new KeeperException.DataInconsistencyException();
+ ke.initCause(e);
+ return ke;
}
}
\ No newline at end of file
Modified: hbase/trunk/src/main/protobuf/ZooKeeper.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/protobuf/ZooKeeper.proto?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/main/protobuf/ZooKeeper.proto (original)
+++ hbase/trunk/src/main/protobuf/ZooKeeper.proto Wed May 2 16:26:36 2012
@@ -60,3 +60,33 @@ message ClusterUp {
// the data is cluster startDate.
required string startDate = 1;
}
+
+/**
+ * What we write under unassigned up in zookeeper as a region moves through
+ * open/close, etc., regions. Details a region in transition.
+ */
+message RegionTransition {
+ // Code for EventType gotten by doing o.a.h.h.EventHandler.EventType.getCode()
+ required uint32 eventTypeCode = 1;
+ // Full regionname in bytes
+ required bytes regionName = 2;
+ required uint64 createTime = 3;
+ optional ServerName originServerName = 4;
+ optional bytes payload = 5;
+}
+
+/**
+ * WAL SplitLog directory znodes have this for content. Used doing distributed
+ * WAL splitting. Holds current state and name of server that originated split.
+ */
+message SplitLogTask {
+ enum State {
+ UNASSIGNED = 0;
+ OWNED = 1;
+ RESIGNED = 2;
+ DONE = 3;
+ ERR = 4;
+ }
+ required State state = 1;
+ required ServerName serverName = 2;
+}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Wed May 2 16:26:36 2012
@@ -81,6 +81,7 @@ import org.apache.hadoop.hbase.util.JVMC
import org.apache.hadoop.hbase.util.RegionSplitter;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestSerialization.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestSerialization.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestSerialization.java Wed May 2 16:26:36 2012
@@ -59,6 +59,14 @@ import org.junit.experimental.categories
@Category(SmallTests.class)
public class TestSerialization {
@Test
+ public void testSplitLogTask() throws DeserializationException {
+ SplitLogTask slt = new SplitLogTask.Unassigned(new ServerName("mgr,1,1"));
+ byte [] bytes = slt.toByteArray();
+ SplitLogTask sltDeserialized = SplitLogTask.parseFrom(bytes);
+ assertTrue(slt.equals(sltDeserialized));
+ }
+
+ @Test
public void testHServerLoadVersioning() throws IOException {
Set<String> cps = new HashSet<String>(0);
Map<byte [], RegionLoad> regions = new TreeMap<byte [], RegionLoad>(Bytes.BYTES_COMPARATOR);
@@ -66,7 +74,7 @@ public class TestSerialization {
new HServerLoad092.RegionLoad(HConstants.META_TABLE_NAME, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, cps));
HServerLoad092 hsl092 = new HServerLoad092(0, 0, 0, 0, regions, cps);
byte [] hsl092bytes = Writables.getBytes(hsl092);
- HServerLoad hsl = (HServerLoad)Writables.getWritable(hsl092bytes, new HServerLoad());
+ Writables.getWritable(hsl092bytes, new HServerLoad());
// TO BE CONTINUED
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java Wed May 2 16:26:36 2012
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -154,8 +155,8 @@ public class TestZooKeeper {
// It's surprising but sometimes we can still be in connected state.
// As it's known (even if not understood) we don't make the the test fail
- // for this reason.
- Assert.assertTrue(state == States.CLOSED);
+ // for this reason.)
+ // Assert.assertTrue("state=" + state, state == States.CLOSED);
// Check that the client recovered
ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
@@ -179,7 +180,7 @@ public class TestZooKeeper {
connection.close();
}
-
+
@Test
public void testRegionServerSessionExpired() throws Exception {
LOG.info("Starting testRegionServerSessionExpired");
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java Wed May 2 16:26:36 2012
@@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -51,7 +50,7 @@ public class TestMetaReaderEditor {
private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static ZooKeeperWatcher zkw;
- private static CatalogTracker ct;
+ private static CatalogTracker CT;
private final static Abortable ABORTABLE = new Abortable() {
private final AtomicBoolean abort = new AtomicBoolean(false);
@@ -77,13 +76,13 @@ public class TestMetaReaderEditor {
c.setLong("hbase.client.pause", 1000);
c.setInt("hbase.client.retries.number", 10);
zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE);
- ct = new CatalogTracker(zkw, c, ABORTABLE);
- ct.start();
+ CT = new CatalogTracker(zkw, c, ABORTABLE);
+ CT.start();
}
@AfterClass public static void afterClass() throws Exception {
ABORTABLE.abort("test ending", null);
- ct.stop();
+ CT.stop();
UTIL.shutdownMiniCluster();
}
@@ -103,15 +102,15 @@ public class TestMetaReaderEditor {
int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
// Test it works getting a region from just made user table.
final List<HRegionInfo> regions =
- testGettingTableRegions(this.ct, nameBytes, regionCount);
- MetaTask reader = new MetaTask(this.ct, "reader") {
+ testGettingTableRegions(CT, nameBytes, regionCount);
+ MetaTask reader = new MetaTask(CT, "reader") {
@Override
void metaTask() throws Throwable {
testGetRegion(this.ct, regions.get(0));
LOG.info("Read " + regions.get(0).getEncodedName());
}
};
- MetaTask writer = new MetaTask(this.ct, "writer") {
+ MetaTask writer = new MetaTask(CT, "writer") {
@Override
void metaTask() throws Throwable {
MetaEditor.addRegionToMeta(this.ct, regions.get(0));
@@ -140,7 +139,7 @@ public class TestMetaReaderEditor {
int index = -1;
do {
index = UTIL.getMiniHBaseCluster().getServerWithMeta();
- }while (index == -1 &&
+ } while (index == -1 &&
startTime + timeOut < System.currentTimeMillis());
if (index != -1){
@@ -149,8 +148,8 @@ public class TestMetaReaderEditor {
}
}
- assertTrue("reader: "+reader.toString(), reader.isProgressing());
- assertTrue("writer: "+writer.toString(), writer.isProgressing());
+ assertTrue("reader: " + reader.toString(), reader.isProgressing());
+ assertTrue("writer: " + writer.toString(), writer.isProgressing());
} catch (IOException e) {
throw e;
} finally {
@@ -216,27 +215,27 @@ public class TestMetaReaderEditor {
@Test public void testGetRegionsCatalogTables()
throws IOException, InterruptedException {
List<HRegionInfo> regions =
- MetaReader.getTableRegions(ct, HConstants.META_TABLE_NAME);
+ MetaReader.getTableRegions(CT, HConstants.META_TABLE_NAME);
assertTrue(regions.size() >= 1);
- assertTrue(MetaReader.getTableRegionsAndLocations(ct,
+ assertTrue(MetaReader.getTableRegionsAndLocations(CT,
Bytes.toString(HConstants.META_TABLE_NAME)).size() >= 1);
- assertTrue(MetaReader.getTableRegionsAndLocations(ct,
+ assertTrue(MetaReader.getTableRegionsAndLocations(CT,
Bytes.toString(HConstants.ROOT_TABLE_NAME)).size() == 1);
}
@Test public void testTableExists() throws IOException {
final String name = "testTableExists";
final byte [] nameBytes = Bytes.toBytes(name);
- assertFalse(MetaReader.tableExists(ct, name));
+ assertFalse(MetaReader.tableExists(CT, name));
UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
- assertTrue(MetaReader.tableExists(ct, name));
+ assertTrue(MetaReader.tableExists(CT, name));
HBaseAdmin admin = UTIL.getHBaseAdmin();
admin.disableTable(name);
admin.deleteTable(name);
- assertFalse(MetaReader.tableExists(ct, name));
- assertTrue(MetaReader.tableExists(ct,
+ assertFalse(MetaReader.tableExists(CT, name));
+ assertTrue(MetaReader.tableExists(CT,
Bytes.toString(HConstants.META_TABLE_NAME)));
- assertTrue(MetaReader.tableExists(ct,
+ assertTrue(MetaReader.tableExists(CT,
Bytes.toString(HConstants.ROOT_TABLE_NAME)));
}
@@ -245,11 +244,11 @@ public class TestMetaReaderEditor {
LOG.info("Started " + name);
// Test get on non-existent region.
Pair<HRegionInfo, ServerName> pair =
- MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region"));
+ MetaReader.getRegion(CT, Bytes.toBytes("nonexistent-region"));
assertNull(pair);
// Test it works getting a region from meta/root.
pair =
- MetaReader.getRegion(ct, HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+ MetaReader.getRegion(CT, HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
assertEquals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(),
pair.getFirst().getEncodedName());
LOG.info("Finished " + name);
@@ -273,8 +272,8 @@ public class TestMetaReaderEditor {
// Now make sure we only get the regions from 1 of the tables at a time
- assertEquals(1, MetaReader.getTableRegions(ct, Bytes.toBytes(name)).size());
- assertEquals(1, MetaReader.getTableRegions(ct, greaterName).size());
+ assertEquals(1, MetaReader.getTableRegions(CT, Bytes.toBytes(name)).size());
+ assertEquals(1, MetaReader.getTableRegions(CT, greaterName).size());
}
private static List<HRegionInfo> testGettingTableRegions(final CatalogTracker ct,
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java Wed May 2 16:26:36 2012
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.security.
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -79,7 +78,6 @@ public class TestWALObserver {
private Configuration conf;
private FileSystem fs;
private Path dir;
- private MiniDFSCluster cluster;
private Path hbaseRootDir;
private Path oldLogDir;
private Path logDir;
@@ -136,8 +134,6 @@ public class TestWALObserver {
HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
- HRegion region2 = HRegion.createHRegion(hri,
- hbaseRootDir, this.conf, htd);
Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
deleteDir(basedir);
@@ -236,10 +232,6 @@ public class TestWALObserver {
final Configuration newConf = HBaseConfiguration.create(this.conf);
- HRegion region2 = HRegion.createHRegion(hri,
- hbaseRootDir, newConf,htd);
-
-
//HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
HLog wal = createWAL(this.conf);
//Put p = creatPutWith2Families(TEST_ROW);
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/Mocking.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/Mocking.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/Mocking.java Wed May 2 16:26:36 2012
@@ -23,9 +23,11 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.executor.EventHandler.EventType;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
@@ -71,12 +74,13 @@ public class Mocking {
* @param sn Name of the regionserver doing the 'opening'
* @param hri Region we're 'opening'.
* @throws KeeperException
+ * @throws DeserializationException
*/
static void fakeRegionServerRegionOpenInZK(final ZooKeeperWatcher w,
final ServerName sn, final HRegionInfo hri)
- throws KeeperException {
+ throws KeeperException, DeserializationException {
// Wait till we see the OFFLINE zk node before we proceed.
- while (!ZKAssign.verifyRegionState(w, hri, EventType.M_ZK_REGION_OFFLINE)) {
+ while (!verifyRegionState(w, hri, EventType.M_ZK_REGION_OFFLINE)) {
Threads.sleep(1);
}
// Get current versionid else will fail on transition from OFFLINE to OPENING below
@@ -94,4 +98,40 @@ public class Mocking {
// We should be done now. The master open handler will notice the
// transition and remove this regions znode.
}
+
+ /**
+ * Verifies that the specified region is in the specified state in ZooKeeper.
+ * <p>
+ * Returns true if region is in transition and in the specified state in
+ * ZooKeeper. Returns false if the region does not exist in ZK or is in
+ * a different state.
+ * <p>
+ * Method synchronizes() with ZK so will yield an up-to-date result but is
+ * a slow read.
+ * @param zkw
+ * @param region
+ * @param expectedState
+ * @return true if region exists and is in expected state
+ * @throws DeserializationException
+ */
+ static boolean verifyRegionState(ZooKeeperWatcher zkw, HRegionInfo region, EventType expectedState)
+ throws KeeperException, DeserializationException {
+ String encoded = region.getEncodedName();
+
+ String node = ZKAssign.getNodeName(zkw, encoded);
+ zkw.sync(node);
+
+ // Read existing data of the node
+ byte [] existingBytes = null;
+ try {
+ existingBytes = ZKUtil.getDataAndWatch(zkw, node);
+ } catch (KeeperException.NoNodeException nne) {
+ return false;
+ } catch (KeeperException e) {
+ throw e;
+ }
+ if (existingBytes == null) return false;
+ RegionTransition rt = RegionTransition.parseFrom(existingBytes);
+ return rt.getEventType().equals(expectedState);
+ }
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java Wed May 2 16:26:36 2012
@@ -29,23 +29,24 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.client.ClientProtocol;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -61,10 +62,9 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hbase.MediumTests;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.Watcher;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -115,6 +115,7 @@ public class TestAssignmentManager {
// If abort is called, be sure to fail the test (don't just swallow it
// silently as is mockito default).
this.server = Mockito.mock(Server.class);
+ Mockito.when(server.getServerName()).thenReturn(new ServerName("master,1,1"));
Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
this.watcher =
new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
@@ -159,10 +160,11 @@ public class TestAssignmentManager {
* @throws IOException
* @throws KeeperException
* @throws InterruptedException
+ * @throws DeserializationException
*/
@Test(timeout = 5000)
public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
- throws IOException, KeeperException, InterruptedException, ServiceException {
+ throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
AssignmentManagerWithExtrasForTesting am =
setUpMockedAssignmentManager(this.server, this.serverManager);
try {
@@ -177,7 +179,7 @@ public class TestAssignmentManager {
int versionid =
ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
assertNotSame(versionid, -1);
- while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
+ while (!Mocking.verifyRegionState(this.watcher, REGIONINFO,
EventType.M_ZK_REGION_OFFLINE)) {
Threads.sleep(1);
}
@@ -205,7 +207,7 @@ public class TestAssignmentManager {
@Test(timeout = 5000)
public void testBalanceOnMasterFailoverScenarioWithClosedNode()
- throws IOException, KeeperException, InterruptedException, ServiceException {
+ throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
AssignmentManagerWithExtrasForTesting am =
setUpMockedAssignmentManager(this.server, this.serverManager);
try {
@@ -221,7 +223,7 @@ public class TestAssignmentManager {
ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
assertNotSame(versionid, -1);
am.gate.set(false);
- while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
+ while (!Mocking.verifyRegionState(this.watcher, REGIONINFO,
EventType.M_ZK_REGION_OFFLINE)) {
Threads.sleep(1);
}
@@ -249,7 +251,7 @@ public class TestAssignmentManager {
@Test(timeout = 5000)
public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
- throws IOException, KeeperException, InterruptedException, ServiceException {
+ throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
AssignmentManagerWithExtrasForTesting am =
setUpMockedAssignmentManager(this.server, this.serverManager);
try {
@@ -264,7 +266,7 @@ public class TestAssignmentManager {
int versionid =
ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
assertNotSame(versionid, -1);
- while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
+ while (!Mocking.verifyRegionState(this.watcher, REGIONINFO,
EventType.M_ZK_REGION_OFFLINE)) {
Threads.sleep(1);
}
@@ -306,10 +308,11 @@ public class TestAssignmentManager {
* from one server to another mocking regionserver responding over zk.
* @throws IOException
* @throws KeeperException
+ * @throws DeserializationException
*/
@Test
public void testBalance()
- throws IOException, KeeperException {
+ throws IOException, KeeperException, DeserializationException {
// Create and startup an executor. This is used by AssignmentManager
// handling zk callbacks.
ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
@@ -345,7 +348,7 @@ public class TestAssignmentManager {
// balancer. The zk node will be OFFLINE waiting for regionserver to
// transition it through OPENING, OPENED. Wait till we see the OFFLINE
// zk node before we proceed.
- while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO, EventType.M_ZK_REGION_OFFLINE)) {
+ while (!Mocking.verifyRegionState(this.watcher, REGIONINFO, EventType.M_ZK_REGION_OFFLINE)) {
Threads.sleep(1);
}
// Get current versionid else will fail on transition from OFFLINE to OPENING below
@@ -541,12 +544,12 @@ public class TestAssignmentManager {
private static int createNodeSplitting(final ZooKeeperWatcher zkw,
final HRegionInfo region, final ServerName serverName)
throws KeeperException, IOException {
- RegionTransitionData data =
- new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
+ RegionTransition rt =
+ RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
region.getRegionName(), serverName);
String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
- if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
+ if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
throw new IOException("Failed create of ephemeral " + node);
}
// Transition node from SPLITTING to SPLITTING and pick up version so we
@@ -650,12 +653,12 @@ public class TestAssignmentManager {
deadServers);
}
@Override
- void processRegionsInTransition(final RegionTransitionData data,
+ void processRegionsInTransition(final RegionTransition rt,
final HRegionInfo regionInfo,
final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
final int expectedVersion) throws KeeperException {
while (this.gate.get()) Threads.sleep(1);
- super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
+ super.processRegionsInTransition(rt, regionInfo, deadServers, expectedVersion);
}
/** reset the watcher */
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java Wed May 2 16:26:36 2012
@@ -19,7 +19,7 @@
*/
package org.apache.hadoop.hbase.master;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+import static org.apache.hadoop.hbase.SplitLogCounters.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.util.FSUt
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
-import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -73,7 +72,7 @@ public class TestDistributedLogSplitting
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
}
- // Start a cluster with 2 masters and 3 regionservers
+ // Start a cluster with 2 masters and 6 regionservers
final int NUM_MASTERS = 2;
final int NUM_RS = 6;
@@ -83,7 +82,7 @@ public class TestDistributedLogSplitting
HBaseTestingUtility TEST_UTIL;
private void startCluster(int num_rs) throws Exception{
- ZKSplitLog.Counters.resetCounters();
+ SplitLogCounters.resetCounters();
LOG.info("Starting cluster");
conf = HBaseConfiguration.create();
conf.getLong("hbase.splitlog.max.resubmit", 0);
@@ -109,48 +108,6 @@ public class TestDistributedLogSplitting
}
@Test (timeout=300000)
- public void testThreeRSAbort() throws Exception {
- LOG.info("testThreeRSAbort");
- final int NUM_REGIONS_TO_CREATE = 40;
- final int NUM_ROWS_PER_REGION = 100;
-
- startCluster(NUM_RS); // NUM_RS=6.
-
- ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
- "distributed log splitting test", null);
-
- HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
- populateDataInTable(NUM_ROWS_PER_REGION, "family");
-
-
- List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
- assertEquals(NUM_RS, rsts.size());
- rsts.get(0).getRegionServer().abort("testing");
- rsts.get(1).getRegionServer().abort("testing");
- rsts.get(2).getRegionServer().abort("testing");
-
- long start = EnvironmentEdgeManager.currentTimeMillis();
- while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
- if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
- assertTrue(false);
- }
- Thread.sleep(200);
- }
-
- start = EnvironmentEdgeManager.currentTimeMillis();
- while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
- if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
- assertTrue(false);
- }
- Thread.sleep(200);
- }
-
- assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
- TEST_UTIL.countRows(ht));
- ht.close();
- }
-
- @Test (timeout=300000)
public void testRecoveredEdits() throws Exception {
LOG.info("testRecoveredEdits");
startCluster(NUM_RS);
@@ -196,8 +153,7 @@ public class TestDistributedLogSplitting
Path tdir = HTableDescriptor.getTableDir(rootdir, table);
Path editsdir =
- HLog.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir,
- hri.getEncodedName()));
+ HLog.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
LOG.debug("checking edits dir " + editsdir);
FileStatus[] files = fs.listStatus(editsdir);
assertEquals(1, files.length);
@@ -272,6 +228,50 @@ public class TestDistributedLogSplitting
"tot_wkr_preempt_task");
}
+ @Test (timeout=300000)
+ public void testThreeRSAbort() throws Exception {
+ LOG.info("testThreeRSAbort");
+ final int NUM_REGIONS_TO_CREATE = 40;
+ final int NUM_ROWS_PER_REGION = 100;
+
+ startCluster(NUM_RS); // NUM_RS=6.
+
+ ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
+ "distributed log splitting test", null);
+
+ HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+ populateDataInTable(NUM_ROWS_PER_REGION, "family");
+
+
+ List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+ assertEquals(NUM_RS, rsts.size());
+ rsts.get(0).getRegionServer().abort("testing");
+ rsts.get(1).getRegionServer().abort("testing");
+ rsts.get(2).getRegionServer().abort("testing");
+
+ long start = EnvironmentEdgeManager.currentTimeMillis();
+ while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
+ if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+ assertTrue(false);
+ }
+ Thread.sleep(200);
+ }
+
+ start = EnvironmentEdgeManager.currentTimeMillis();
+ while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
+ if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+ assertTrue("Timedout", false);
+ }
+ Thread.sleep(200);
+ }
+
+ assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
+ TEST_UTIL.countRows(ht));
+ ht.close();
+ }
+
+
+
@Test(timeout=25000)
public void testDelayedDeleteOnFailure() throws Exception {
LOG.info("testDelayedDeleteOnFailure");
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java Wed May 2 16:26:36 2012
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.executor.EventHandler.EventType;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -57,167 +56,6 @@ import org.junit.experimental.categories
public class TestMasterFailover {
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
- @Test (timeout=180000)
- public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
- throws Exception {
- LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
- final int NUM_MASTERS = 1;
- final int NUM_RS = 2;
-
- Configuration conf = HBaseConfiguration.create();
- conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
- conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
- // Start the cluster
- HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
-
- TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-
- // Find regionserver carrying meta.
- List<RegionServerThread> regionServerThreads =
- cluster.getRegionServerThreads();
- int count = -1;
- HRegion metaRegion = null;
- for (RegionServerThread regionServerThread : regionServerThreads) {
- HRegionServer regionServer = regionServerThread.getRegionServer();
- metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
- count++;
- regionServer.abort("");
- if (null != metaRegion) break;
- }
- HRegionServer regionServer = cluster.getRegionServer(count);
-
- TEST_UTIL.shutdownMiniHBaseCluster();
-
- // Create a ZKW to use in the test
- ZooKeeperWatcher zkw =
- HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
- metaRegion, regionServer.getServerName());
-
- LOG.info("Staring cluster for second time");
- TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
-
- // Failover should be completed, now wait for no RIT
- log("Waiting for no more RIT");
- ZKAssign.blockUntilNoRIT(zkw);
-
- zkw.close();
- // Stop the cluster
- TEST_UTIL.shutdownMiniCluster();
- }
-
- /**
- * Simple test of master failover.
- * <p>
- * Starts with three masters. Kills a backup master. Then kills the active
- * master. Ensures the final master becomes active and we can still contact
- * the cluster.
- * @throws Exception
- */
- @Test (timeout=240000)
- public void testSimpleMasterFailover() throws Exception {
-
- final int NUM_MASTERS = 3;
- final int NUM_RS = 3;
-
- // Create config to use for this cluster
- Configuration conf = HBaseConfiguration.create();
-
- // Start the cluster
- HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
- TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-
- // get all the master threads
- List<MasterThread> masterThreads = cluster.getMasterThreads();
-
- // wait for each to come online
- for (MasterThread mt : masterThreads) {
- assertTrue(mt.isAlive());
- }
-
- // verify only one is the active master and we have right number
- int numActive = 0;
- int activeIndex = -1;
- ServerName activeName = null;
- HMaster active = null;
- for (int i = 0; i < masterThreads.size(); i++) {
- if (masterThreads.get(i).getMaster().isActiveMaster()) {
- numActive++;
- activeIndex = i;
- active = masterThreads.get(activeIndex).getMaster();
- activeName = active.getServerName();
- }
- }
- assertEquals(1, numActive);
- assertEquals(NUM_MASTERS, masterThreads.size());
- LOG.info("Active master " + activeName);
-
- // Check that ClusterStatus reports the correct active and backup masters
- assertNotNull(active);
- ClusterStatus status = active.getClusterStatus();
- assertTrue(status.getMaster().equals(activeName));
- assertEquals(2, status.getBackupMastersSize());
- assertEquals(2, status.getBackupMasters().size());
-
- // attempt to stop one of the inactive masters
- int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
- HMaster master = cluster.getMaster(backupIndex);
- LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
- cluster.stopMaster(backupIndex, false);
- cluster.waitOnMaster(backupIndex);
-
- // Verify still one active master and it's the same
- for (int i = 0; i < masterThreads.size(); i++) {
- if (masterThreads.get(i).getMaster().isActiveMaster()) {
- assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
- activeIndex = i;
- active = masterThreads.get(activeIndex).getMaster();
- }
- }
- assertEquals(1, numActive);
- assertEquals(2, masterThreads.size());
- int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
- LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers");
- assertEquals(3, rsCount);
-
- // Check that ClusterStatus reports the correct active and backup masters
- assertNotNull(active);
- status = active.getClusterStatus();
- assertTrue(status.getMaster().equals(activeName));
- assertEquals(1, status.getBackupMastersSize());
- assertEquals(1, status.getBackupMasters().size());
-
- // kill the active master
- LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
- cluster.stopMaster(activeIndex, false);
- cluster.waitOnMaster(activeIndex);
-
- // wait for an active master to show up and be ready
- assertTrue(cluster.waitForActiveAndReadyMaster());
-
- LOG.debug("\n\nVerifying backup master is now active\n");
- // should only have one master now
- assertEquals(1, masterThreads.size());
-
- // and he should be active
- active = masterThreads.get(0).getMaster();
- assertNotNull(active);
- status = active.getClusterStatus();
- ServerName mastername = status.getMaster();
- assertTrue(mastername.equals(active.getServerName()));
- assertTrue(active.isActiveMaster());
- assertEquals(0, status.getBackupMastersSize());
- assertEquals(0, status.getBackupMasters().size());
- int rss = status.getServersSize();
- LOG.info("Active master " + mastername.getServerName() + " managing " +
- rss + " region servers");
- assertEquals(3, rss);
-
- // Stop the cluster
- TEST_UTIL.shutdownMiniCluster();
- }
-
/**
* Complex test of master failover that tests as many permutations of the
* different possible states that regions in transition could be in within ZK.
@@ -475,8 +313,9 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, serverName);
hrs.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
@@ -488,8 +327,9 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, serverName);
hrs.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
@@ -603,16 +443,14 @@ public class TestMasterFailover {
final int NUM_MASTERS = 1;
final int NUM_RS = 2;
- // Create config to use for this cluster
- Configuration conf = HBaseConfiguration.create();
+ // Create and start the cluster
+ HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ Configuration conf = TEST_UTIL.getConfiguration();
// Need to drop the timeout much lower
conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
conf.setInt("hbase.master.wait.on.regionservers.mintostart", 1);
conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 2);
-
- // Create and start the cluster
- HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Cluster started");
@@ -835,8 +673,9 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, deadServerName);
hrsDead.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
@@ -850,8 +689,9 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, deadServerName);
hrsDead.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
@@ -869,9 +709,11 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, deadServerName);
hrsDead.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
+ LOG.debug("DELETED " + rt);
break;
}
Thread.sleep(100);
@@ -885,8 +727,9 @@ public class TestMasterFailover {
ZKAssign.createNodeOffline(zkw, region, deadServerName);
hrsDead.openRegion(region);
while (true) {
- RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
- if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+ byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
+ RegionTransition rt = RegionTransition.parseFrom(bytes);
+ if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
break;
}
@@ -984,7 +827,8 @@ public class TestMasterFailover {
// Now, everything that should be online should be online
for (HRegionInfo hri : regionsThatShouldBeOnline) {
- assertTrue("region=" + hri.getRegionNameAsString(), onlineRegions.contains(hri));
+ assertTrue("region=" + hri.getRegionNameAsString() + ", " + onlineRegions.toString(),
+ onlineRegions.contains(hri));
}
// Everything that should be offline should not be online
@@ -1018,6 +862,167 @@ public class TestMasterFailover {
LOG.info("\n\n" + string + " \n\n");
}
+ @Test (timeout=180000)
+ public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
+ throws Exception {
+ LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
+ final int NUM_MASTERS = 1;
+ final int NUM_RS = 2;
+
+ // Start the cluster
+ HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ Configuration conf = TEST_UTIL.getConfiguration();
+ conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
+ conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
+ conf.setInt("hbase.master.info.port", -1);
+
+ TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+
+ // Find regionserver carrying meta.
+ List<RegionServerThread> regionServerThreads =
+ cluster.getRegionServerThreads();
+ int count = -1;
+ HRegion metaRegion = null;
+ for (RegionServerThread regionServerThread : regionServerThreads) {
+ HRegionServer regionServer = regionServerThread.getRegionServer();
+ metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+ count++;
+ regionServer.abort("");
+ if (null != metaRegion) break;
+ }
+ HRegionServer regionServer = cluster.getRegionServer(count);
+
+ TEST_UTIL.shutdownMiniHBaseCluster();
+
+ // Create a ZKW to use in the test
+ ZooKeeperWatcher zkw =
+ HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
+ metaRegion, regionServer.getServerName());
+
+ LOG.info("Staring cluster for second time");
+ TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
+
+ // Failover should be completed, now wait for no RIT
+ log("Waiting for no more RIT");
+ ZKAssign.blockUntilNoRIT(zkw);
+
+ zkw.close();
+ // Stop the cluster
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Simple test of master failover.
+ * <p>
+ * Starts with three masters. Kills a backup master. Then kills the active
+ * master. Ensures the final master becomes active and we can still contact
+ * the cluster.
+ * @throws Exception
+ */
+ @Test (timeout=240000)
+ public void testSimpleMasterFailover() throws Exception {
+
+ final int NUM_MASTERS = 3;
+ final int NUM_RS = 3;
+
+ // Start the cluster
+ HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+
+ // get all the master threads
+ List<MasterThread> masterThreads = cluster.getMasterThreads();
+
+ // wait for each to come online
+ for (MasterThread mt : masterThreads) {
+ assertTrue(mt.isAlive());
+ }
+
+ // verify only one is the active master and we have right number
+ int numActive = 0;
+ int activeIndex = -1;
+ ServerName activeName = null;
+ HMaster active = null;
+ for (int i = 0; i < masterThreads.size(); i++) {
+ if (masterThreads.get(i).getMaster().isActiveMaster()) {
+ numActive++;
+ activeIndex = i;
+ active = masterThreads.get(activeIndex).getMaster();
+ activeName = active.getServerName();
+ }
+ }
+ assertEquals(1, numActive);
+ assertEquals(NUM_MASTERS, masterThreads.size());
+ LOG.info("Active master " + activeName);
+
+ // Check that ClusterStatus reports the correct active and backup masters
+ assertNotNull(active);
+ ClusterStatus status = active.getClusterStatus();
+ assertTrue(status.getMaster().equals(activeName));
+ assertEquals(2, status.getBackupMastersSize());
+ assertEquals(2, status.getBackupMasters().size());
+
+ // attempt to stop one of the inactive masters
+ int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
+ HMaster master = cluster.getMaster(backupIndex);
+ LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
+ cluster.stopMaster(backupIndex, false);
+ cluster.waitOnMaster(backupIndex);
+
+ // Verify still one active master and it's the same
+ for (int i = 0; i < masterThreads.size(); i++) {
+ if (masterThreads.get(i).getMaster().isActiveMaster()) {
+ assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
+ activeIndex = i;
+ active = masterThreads.get(activeIndex).getMaster();
+ }
+ }
+ assertEquals(1, numActive);
+ assertEquals(2, masterThreads.size());
+ int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
+ LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers");
+ assertEquals(3, rsCount);
+
+ // Check that ClusterStatus reports the correct active and backup masters
+ assertNotNull(active);
+ status = active.getClusterStatus();
+ assertTrue(status.getMaster().equals(activeName));
+ assertEquals(1, status.getBackupMastersSize());
+ assertEquals(1, status.getBackupMasters().size());
+
+ // kill the active master
+ LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
+ cluster.stopMaster(activeIndex, false);
+ cluster.waitOnMaster(activeIndex);
+
+ // wait for an active master to show up and be ready
+ assertTrue(cluster.waitForActiveAndReadyMaster());
+
+ LOG.debug("\n\nVerifying backup master is now active\n");
+ // should only have one master now
+ assertEquals(1, masterThreads.size());
+
+ // and he should be active
+ active = masterThreads.get(0).getMaster();
+ assertNotNull(active);
+ status = active.getClusterStatus();
+ ServerName mastername = status.getMaster();
+ assertTrue(mastername.equals(active.getServerName()));
+ assertTrue(active.isActiveMaster());
+ assertEquals(0, status.getBackupMastersSize());
+ assertEquals(0, status.getBackupMasters().size());
+ int rss = status.getServersSize();
+ LOG.info("Active master " + mastername.getServerName() + " managing " +
+ rss + " region servers");
+ assertEquals(3, rss);
+
+ // Stop the cluster
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java?rev=1333099&r1=1333098&r2=1333099&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java Wed May 2 16:26:36 2012
@@ -27,6 +27,7 @@ import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -227,10 +228,11 @@ public class TestMasterNoCluster {
* @throws IOException
* @throws KeeperException
* @throws InterruptedException
+ * @throws DeserializationException
*/
@Test
public void testCatalogDeploys()
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, DeserializationException {
final Configuration conf = TESTUTIL.getConfiguration();
final long now = System.currentTimeMillis();
// Name for our single mocked up regionserver.