You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2014/06/14 17:40:40 UTC
[5/5] git commit: HBASE-11059 ZK-less region assignment
HBASE-11059 ZK-less region assignment
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58549428
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58549428
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58549428
Branch: refs/heads/master
Commit: 58549428a66550aed83bfb6f1da405decc1b0f61
Parents: 047463c
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Wed Apr 23 16:56:36 2014 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Sat Jun 14 08:39:29 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 1 +
.../org/apache/hadoop/hbase/RegionLoad.java | 4 +-
.../apache/hadoop/hbase/catalog/MetaReader.java | 10 +-
.../apache/hadoop/hbase/master/RegionState.java | 37 +-
.../hbase/zookeeper/ZooKeeperWatcher.java | 4 +-
.../org/apache/hadoop/hbase/HConstants.java | 10 +
.../generated/RegionServerStatusProtos.java | 2959 +++++++++++++++++-
.../src/main/protobuf/RegionServerStatus.proto | 49 +
.../hadoop/hbase/master/AssignCallable.java | 7 +-
.../hadoop/hbase/master/AssignmentManager.java | 782 ++++-
.../hadoop/hbase/master/MasterRpcServices.java | 35 +-
.../hadoop/hbase/master/RegionStateStore.java | 226 ++
.../hadoop/hbase/master/RegionStates.java | 295 +-
.../hadoop/hbase/master/ServerManager.java | 12 +-
.../hadoop/hbase/master/UnAssignCallable.java | 47 +
.../master/handler/ServerShutdownHandler.java | 27 +-
.../hbase/regionserver/HRegionServer.java | 94 +-
.../regionserver/RegionMergeTransaction.java | 111 +-
.../regionserver/RegionServerServices.java | 11 +
.../hbase/regionserver/SplitTransaction.java | 53 +-
.../handler/CloseRegionHandler.java | 15 +-
.../regionserver/handler/OpenRegionHandler.java | 46 +-
.../apache/hadoop/hbase/util/ConfigUtil.java | 33 +
.../hadoop/hbase/MockRegionServerServices.java | 15 +-
.../apache/hadoop/hbase/TestDrainingServer.java | 1 +
.../client/TestScannersFromClientSide.java | 5 +-
.../hadoop/hbase/master/MockRegionServer.java | 14 +-
.../hbase/master/TestAssignmentManager.java | 4 +-
.../master/TestAssignmentManagerOnCluster.java | 42 +-
.../hadoop/hbase/master/TestMasterFailover.java | 104 +
.../hbase/master/TestOpenedRegionHandler.java | 2 +
.../hadoop/hbase/master/TestRestartCluster.java | 1 +
.../master/TestZKBasedOpenCloseRegion.java | 1 +
.../hbase/master/TestZKLessAMOnCluster.java | 42 +
.../TestEndToEndSplitTransaction.java | 8 +-
.../TestRegionMergeTransactionOnCluster.java | 20 +-
.../regionserver/TestRegionServerNoMaster.java | 1 +
.../TestSplitTransactionOnCluster.java | 134 +-
.../regionserver/TestZKLessMergeOnCluster.java | 45 +
.../regionserver/TestZKLessSplitOnCluster.java | 45 +
.../handler/TestCloseRegionHandler.java | 1 +
.../handler/TestOpenRegionHandler.java | 1 +
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 6 +
.../zookeeper/TestRecoverableZooKeeper.java | 2 +-
44 files changed, 4907 insertions(+), 455 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 78e175d..cc87c49 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -1048,6 +1048,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* @param r Result to pull from
* @return A ServerName instance or null if necessary fields not found or empty.
*/
+ // TODO: ServerName should be a better place for this method?
public static ServerName getServerName(final Result r) {
Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
if (cell == null || cell.getValueLength() == 0) return null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index 2fa45e4..28e6d6b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -203,6 +203,8 @@ public class RegionLoad {
}
sb = Strings.appendKeyValue(sb, "compactionProgressPct",
compactionProgressPct);
+ sb = Strings.appendKeyValue(sb, "completeSequenceId",
+ this.getCompleteSequenceId());
return sb.toString();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 4f15390..ec0f4a0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -160,6 +160,7 @@ public class MetaReader {
* @return An {@link HTable} for <code>tableName</code>
* @throws IOException
*/
+ @SuppressWarnings("deprecation")
private static HTable getHTable(final CatalogTracker catalogTracker,
final TableName tableName)
throws IOException {
@@ -482,8 +483,11 @@ public class MetaReader {
@Override
void add(Result r) {
if (r == null || r.isEmpty()) return;
+ if (HRegionInfo.getHRegionInfo(r) == null) return;
ServerName sn = HRegionInfo.getServerName(r);
- if (sn != null && sn.equals(serverName)) this.results.add(r);
+ if (sn != null && sn.equals(serverName)) {
+ this.results.add(r);
+ }
}
};
fullScan(catalogTracker, v);
@@ -491,8 +495,8 @@ public class MetaReader {
if (results != null && !results.isEmpty()) {
// Convert results to Map keyed by HRI
for (Result r: results) {
- Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
- if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
+ HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
+ if (hri != null) hris.put(hri, r);
}
}
return hris;
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
index 1170387..3289ac1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hbase.master;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;
@@ -35,7 +32,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
* So it is almost immutable.
*/
@InterfaceAudience.Private
-public class RegionState implements org.apache.hadoop.io.Writable {
+public class RegionState {
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -77,6 +74,11 @@ public class RegionState implements org.apache.hadoop.io.Writable {
}
public RegionState(HRegionInfo region,
+ State state, ServerName serverName) {
+ this(region, state, System.currentTimeMillis(), serverName);
+ }
+
+ public RegionState(HRegionInfo region,
State state, long stamp, ServerName serverName) {
this.hri = region;
this.state = state;
@@ -383,25 +385,26 @@ public class RegionState implements org.apache.hadoop.io.Writable {
}
/**
- * @deprecated Writables are going away
+ * Check if two states are the same, except timestamp
*/
- @Deprecated
@Override
- public void readFields(DataInput in) throws IOException {
- hri = new HRegionInfo();
- hri.readFields(in);
- state = State.valueOf(in.readUTF());
- stamp.set(in.readLong());
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ RegionState tmp = (RegionState)obj;
+ return tmp.hri.equals(hri) && tmp.state == state
+ && ((serverName != null && serverName.equals(tmp.serverName))
+ || (tmp.serverName == null && serverName == null));
}
/**
- * @deprecated Writables are going away
+ * Don't count timestamp in hash code calculation
*/
- @Deprecated
@Override
- public void write(DataOutput out) throws IOException {
- hri.write(out);
- out.writeUTF(state.name());
- out.writeLong(stamp.get());
+ public int hashCode() {
+ return (serverName != null ? serverName.hashCode() * 11 : 0)
+ + hri.hashCode() + 5 * state.ordinal();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 52981e4..6b9c8ba 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -171,7 +171,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
try {
// Create all the necessary "directories" of znodes
ZKUtil.createWithParents(this, baseZNode);
- ZKUtil.createAndFailSilent(this, assignmentZNode);
+ if (conf.getBoolean("hbase.assignment.usezk", true)) {
+ ZKUtil.createAndFailSilent(this, assignmentZNode);
+ }
ZKUtil.createAndFailSilent(this, rsZNode);
ZKUtil.createAndFailSilent(this, drainingZNode);
ZKUtil.createAndFailSilent(this, tableZNode);
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3bf1249..b27679c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -405,6 +405,16 @@ public final class HConstants {
/** The open seqnum column qualifier */
public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes("seqnumDuringOpen");
+ /** The state column qualifier */
+ public static final byte [] STATE_QUALIFIER = Bytes.toBytes("state");
+
+ /**
+ * The serverName column qualifier. Its the server where the region is
+ * transitioning on, while column server is the server where the region is
+ * opened on. They are the same when the region is in state OPEN.
+ */
+ public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes("sn");
+
/** The lower-half split region column qualifier */
public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");