You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:07:37 UTC
[01/50] [abbrv] git commit: [HBASE-9704] Fix
TestDrainRegionServer#testDrainAndUndrain
Repository: hbase
Updated Branches:
refs/heads/0.89-fb 9d18d0025 -> a12267f6c
[HBASE-9704] Fix TestDrainRegionServer#testDrainAndUndrain
Summary: HBaseTestingUtility.getTmpKeys() returns an array of byte arrays where the first one is an empty byte array. HBaseTestingUtility.createTable dissallows that.
Test Plan: Ensure TestDrainRegionServer#testDrainAndUndrain passes.
Reviewers: adela, daviddeng, rshroff, liyintang, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1352696
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42583 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d7c39e5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d7c39e5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d7c39e5
Branch: refs/heads/0.89-fb
Commit: 2d7c39e50c8fb087e50f32344daa977af12d125d
Parents: 2d01609
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri May 30 20:56:25 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:43:00 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/client/MetaScanner.java | 12 +++++------
.../hadoop/hbase/HBaseTestingUtility.java | 2 +-
.../hbase/util/TestDrainRegionServer.java | 21 +++++++++++---------
3 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d7c39e5/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 7af1b44..330b8ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -122,10 +122,9 @@ public class MetaScanner {
* will be set to default value <code>Integer.MAX_VALUE</code>.
* @throws IOException e
*/
- public static void metaScan(Configuration configuration, MetaScannerVisitor visitor,
- StringBytes metaTableName, StringBytes tableName, byte[] row,
- int rowLimit)
- throws IOException {
+ public static void metaScan(Configuration configuration,
+ MetaScannerVisitor visitor, StringBytes metaTableName,
+ StringBytes tableName, byte[] row, int rowLimit) throws IOException {
int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
HConnection connection = HConnectionManager.getConnection(configuration);
@@ -142,9 +141,10 @@ public class MetaScanner {
HTable metaTable = new HTable(configuration, metaTableName.getString());
Result startRowResult = metaTable.getRowOrBefore(searchRow,
HConstants.CATALOG_FAMILY);
+ metaTable.close();
if (startRowResult == null) {
- throw new TableNotFoundException("Cannot find row in .META. for table: "
- + tableName + ", row="
+ throw new TableNotFoundException(
+ "Cannot find row in .META. for table: " + tableName + ", row="
+ Bytes.toStringBinary(searchRow));
}
byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d7c39e5/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index f596463..101e552 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -909,7 +909,7 @@ public class HBaseTestingUtility {
public byte[][] getTmpKeys() {
byte[][] KEYS = {
- HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"),
+ Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d7c39e5/src/test/java/org/apache/hadoop/hbase/util/TestDrainRegionServer.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDrainRegionServer.java b/src/test/java/org/apache/hadoop/hbase/util/TestDrainRegionServer.java
index 78f5424..01757c2 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestDrainRegionServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestDrainRegionServer.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.util;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.master.RegionMovementTestHelper;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -48,7 +47,8 @@ import static org.junit.Assert.fail;
@Category(MediumTests.class)
public class TestDrainRegionServer {
- private final static RegionMovementTestHelper TEST_UTIL = new RegionMovementTestHelper();
+ private final static RegionMovementTestHelper TEST_UTIL =
+ new RegionMovementTestHelper();
private static final byte[] FAM_NAM = Bytes.toBytes("f");
private static final int REGION_SERVERS = 5;
@@ -62,11 +62,10 @@ public class TestDrainRegionServer {
@Test
public void testDrainAndUndrain() throws Exception {
-
byte[] tableName = Bytes.toBytes("testBlacklistRegionServerWithoutTimeout");
- HTable table = TEST_UTIL.createTable(
- new StringBytes(tableName), new byte[][] { FAM_NAM } , 1, TEST_UTIL.getTmpKeys());
+ HTable table = TEST_UTIL.createTable(new StringBytes(tableName),
+ new byte[][] { FAM_NAM } , 1, TEST_UTIL.getTmpKeys());
table.close();
List<JVMClusterUtil.RegionServerThread> servers =
@@ -74,7 +73,8 @@ public class TestDrainRegionServer {
AssignmentPlan ap = TEST_UTIL.getHBaseCluster().
getMaster().regionPlacement.getNewAssignmentPlan();
- TEST_UTIL.getHBaseCluster().getMaster().regionPlacement.updateAssignmentPlan(ap);
+ TEST_UTIL.getHBaseCluster().getMaster()
+ .regionPlacement.updateAssignmentPlan(ap);
// Wait for rebalance to to complete
TEST_UTIL.waitOnStableRegionMovement();
@@ -107,19 +107,22 @@ public class TestDrainRegionServer {
int regionsCount = server.getOnlineRegions().size();
- assertTrue("Server to be drained has to have more than 0 regions", regionsCount > 0);
+ assertTrue("Server to be drained has to have more than 0 regions",
+ regionsCount > 0);
rollingRestart.drainServer(drainFile);
TEST_UTIL.waitOnStableRegionMovement();
- assertEquals("Drained server expected to have 0 online regions", 0, server.getOnlineRegions().size());
+ assertEquals("Drained server expected to have 0 online regions", 0,
+ server.getOnlineRegions().size());
rollingRestart.undrainServer(drainFile);
TEST_UTIL.waitOnStableRegionMovement();
- assertEquals("Undrained server expected to have same number of online regions as before",
+ assertEquals("Undrained server expected to have" +
+ "same number of online regions as before",
regionsCount, server.getOnlineRegions().size());
}
[18/50] [abbrv] git commit: [HBASE-11343] Storing the protocol
version information along with the root region server location
Posted by ec...@apache.org.
[HBASE-11343] Storing the protocol version information along with the root region server location
Summary: In order to avoid the configeration overhead of the hbase client jar, the server's protocol version information is stored in the root region server node. This enables us to have hbase bellatrix clients talk to older servers seamlessly.
Test Plan: Added a small unit test to ensure the client can read root region location even if the new znode was deleted and compares the old and new server addresses.
Reviewers: rshroff, mycnyc, elliott, adela
Reviewed By: elliott
Subscribers: andrewcox, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1363451
Tasks: 4449946
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42858 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e14e5d54
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e14e5d54
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e14e5d54
Branch: refs/heads/0.89-fb
Commit: e14e5d5409b4d11756408417307b504ab6f18e65
Parents: 7403a0b
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jun 13 15:58:53 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 8 +
.../apache/hadoop/hbase/ProtocolVersion.java | 28 ++++
.../apache/hadoop/hbase/RootRegionLocation.java | 73 +++++++++
.../hadoop/hbase/client/TableServers.java | 21 ++-
.../hbase/zookeeper/ZooKeeperWrapper.java | 154 +++++++++++++++----
.../client/TestRootRegionLocationZNode.java | 59 +++++++
6 files changed, 308 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index cb140a8..65f7ff8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1116,6 +1116,14 @@ public final class HConstants {
public static final String HSTORE_BLOCKING_WAIT_TIME_KEY =
"hbase.hstore.blockingWaitTime";
public static final long DEFAULT_HSTORE_BLOCKING_WAIT_TIME = 90000;
+
+ public static final ProtocolVersion PROTOCOL_VERSION_THRIFT
+ = ProtocolVersion.THRIFT;
+ public static final ProtocolVersion PROTOCOL_VERSION_HADOOP_RPC
+ = ProtocolVersion.HADOOP_RPC;
+ public static final ProtocolVersion PROTOCOL_VERSION
+ = PROTOCOL_VERSION_THRIFT;
+
private HConstants() {
// Can't be instantiated with this constructor.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/main/java/org/apache/hadoop/hbase/ProtocolVersion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/ProtocolVersion.java b/src/main/java/org/apache/hadoop/hbase/ProtocolVersion.java
new file mode 100644
index 0000000..68726ab
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ProtocolVersion.java
@@ -0,0 +1,28 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import com.facebook.swift.codec.ThriftStruct;
+
+@ThriftStruct
+public enum ProtocolVersion {
+ HADOOP_RPC,
+ THRIFT
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/main/java/org/apache/hadoop/hbase/RootRegionLocation.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/RootRegionLocation.java b/src/main/java/org/apache/hadoop/hbase/RootRegionLocation.java
new file mode 100644
index 0000000..ec2e8e7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/RootRegionLocation.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import com.facebook.swift.codec.ThriftConstructor;
+import com.facebook.swift.codec.ThriftField;
+import com.facebook.swift.codec.ThriftStruct;
+
+@ThriftStruct
+public class RootRegionLocation {
+ private String hostname;
+ private int port;
+ private long startcode;
+ private ProtocolVersion protocolVersion;
+
+ @ThriftConstructor
+ public RootRegionLocation(@ThriftField(1) String hostname,
+ @ThriftField(2) int port,
+ @ThriftField(3) long startcode,
+ @ThriftField(4) ProtocolVersion protocolVersion) {
+ this.hostname = hostname;
+ this.port = port;
+ this.startcode = startcode;
+ this.protocolVersion = protocolVersion;
+ }
+
+ @ThriftField(1)
+ public String getHostName() {
+ return hostname;
+ }
+
+ @ThriftField(2)
+ public int getPort() {
+ return port;
+ }
+
+ @ThriftField(3)
+ public long getStartCode() {
+ return startcode;
+ }
+
+ @ThriftField(4)
+ public ProtocolVersion getProtocolVersion() {
+ return this.protocolVersion;
+ }
+
+ @Override
+ public String toString() {
+ return hostname + ":" + port +
+ ":" + startcode + ":" + protocolVersion.name();
+ }
+
+ public HServerAddress getServerAddress() {
+ return new HServerAddress(hostname, port);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index bb671fb..9c04348 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -65,7 +65,9 @@ import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ProtocolVersion;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.RootRegionLocation;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -243,6 +245,8 @@ public class TableServers implements ServerConnection {
private int batchedUploadSoftFlushRetries;
private long batchedUploadSoftFlushTimeoutMillis;
private final boolean useThrift;
+ // Zookeeper wrapper to query zookeeper for client bootstrap.
+ private static ZooKeeperWrapper staticZK = null;
private Map<StringBytes, StringBytes> initializedTableSet =
new ConcurrentHashMap<>();
@@ -254,8 +258,21 @@ public class TableServers implements ServerConnection {
public TableServers(Configuration conf) {
this.conf = conf;
params = HConnectionParams.getInstance(conf);
- this.useThrift = conf.getBoolean(HConstants.CLIENT_TO_RS_USE_THRIFT,
+ boolean useThrift = conf.getBoolean(HConstants.CLIENT_TO_RS_USE_THRIFT,
HConstants.CLIENT_TO_RS_USE_THRIFT_DEFAULT);
+ try {
+ if (staticZK == null) {
+ staticZK = getZooKeeperWrapper();
+ }
+ RootRegionLocation loc = staticZK.readWrappedRootRegionLocation();
+ if (loc != null) {
+ useThrift = loc.getProtocolVersion() ==
+ ProtocolVersion.THRIFT ? true : false;
+ }
+ } catch (Exception e) {
+ LOG.error("Falling back to config based protocol version detection", e);
+ }
+ this.useThrift = useThrift;
String serverClassName =
conf.get(HConstants.REGION_SERVER_CLASS,
@@ -1279,7 +1296,7 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
*
* @throws IOException
*/
- private HRegionLocation locateRootRegion() throws IOException {
+ protected HRegionLocation locateRootRegion() throws IOException {
// We lazily instantiate the ZooKeeper object because we don't want to
// make the constructor have to throw IOException or handle it itself.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
index 7ba3e0d..de7589e 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
@@ -21,16 +21,12 @@ package org.apache.hadoop.hbase.zookeeper;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
import java.net.Socket;
import java.net.SocketException;
import java.text.DateFormat;
@@ -39,7 +35,6 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -58,6 +53,7 @@ import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.RootRegionLocation;
import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
import org.apache.hadoop.hbase.util.Bytes;
@@ -73,7 +69,6 @@ import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper.States;
import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
/**
@@ -171,6 +166,8 @@ public class ZooKeeperWrapper implements Watcher {
*/
private volatile Abortable abortable;
+ private String rootRegionZNodeFinal;
+
/**
* To allow running multiple independent unit tests within the same JVM, we
* use a concept of "namespaces", which are typically based on test name.
@@ -275,6 +272,9 @@ public class ZooKeeperWrapper implements Watcher {
String rootServerZNodeName =
conf.get("zookeeper.znode.rootserver.complete", "root-region-server-complete");
+
+ String rootRegionZNodeString =
+ conf.get("zookeeper.znode.rootserver.final", "root-region-server-final");
String legacyRootServerZNodeName =
conf.get("zookeeper.znode.rootserver", "root-region-server");
String rsZNodeName = conf.get("zookeeper.znode.rs", "rs");
@@ -283,13 +283,14 @@ public class ZooKeeperWrapper implements Watcher {
String regionsInTransitZNodeName = conf.get("zookeeper.znode.regionInTransition", "UNASSIGNED");
String splitLogZNodeName = conf.get("zookeeper.znode.splitlog", "splitlog");
String tableLockZNodeName = conf.get("zookeeper.znode.tableLock", "tableLock");
- rootRegionZNode = getZNode(parentZNode, rootServerZNodeName);
+ rootRegionZNode = getZNode(parentZNode, rootServerZNodeName);
+ rootRegionZNodeFinal = getZNode(parentZNode, rootRegionZNodeString);
legacyRootRegionZNode = getZNode(parentZNode, legacyRootServerZNodeName);
- rsZNode = getZNode(parentZNode, rsZNodeName);
- rgnsInTransitZNode = getZNode(parentZNode, regionsInTransitZNodeName);
- masterElectionZNode = getZNode(parentZNode, masterAddressZNodeName);
- clusterStateZNode = getZNode(parentZNode, stateZNodeName);
- tableLockZNode = getZNode(parentZNode, tableLockZNodeName);
+ rsZNode = getZNode(parentZNode, rsZNodeName);
+ rgnsInTransitZNode = getZNode(parentZNode, regionsInTransitZNodeName);
+ masterElectionZNode = getZNode(parentZNode, masterAddressZNodeName);
+ clusterStateZNode = getZNode(parentZNode, stateZNodeName);
+ tableLockZNode = getZNode(parentZNode, tableLockZNodeName);
int retryNum = conf.getInt(HConstants.ZOOKEEPER_CONNECTION_RETRY_NUM, 6);
int retryFreq = conf.getInt("zookeeper.connection.retry.freq", 1000);
zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
@@ -376,7 +377,6 @@ public class ZooKeeperWrapper implements Watcher {
}
/** @return String dump of everything in ZooKeeper. */
- @SuppressWarnings({"ConstantConditions"})
public String dump() throws IOException {
StringBuilder sb = new StringBuilder();
@@ -589,18 +589,33 @@ public class ZooKeeperWrapper implements Watcher {
* there was a problem reading the ZNode.
*/
public HServerAddress readRootRegionLocation() {
+ RootRegionLocation loc = readWrappedRootRegionLocation();
+ return loc == null ? null : loc.getServerAddress();
+ }
+
+ public RootRegionLocation readWrappedRootRegionLocation() {
+ RootRegionLocation loc = readRootRegionLocation(rootRegionZNodeFinal, null);
+ if (loc != null) {
+ return loc;
+ }
HServerInfo rootRSInfo = readAddress(rootRegionZNode, null);
if (rootRSInfo == null) {
rootRSInfo = readAddress(legacyRootRegionZNode, null);
}
- return HServerInfo.getAddress(rootRSInfo);
+ if (rootRSInfo == null) return null;
+ HServerAddress addr = HServerInfo.getAddress(rootRSInfo);
+ return new RootRegionLocation(addr.getHostname(), addr.getPort(),
+ rootRSInfo.getStartCode(), HConstants.PROTOCOL_VERSION_HADOOP_RPC);
}
/**
* @return the location of the server serving the root region, including the start code
*/
public HServerInfo readRootRegionServerInfo() {
- return readAddress(rootRegionZNode, null);
+ RootRegionLocation loc = readWrappedRootRegionLocation();
+ if (loc == null) return null;
+ return new HServerInfo(loc.getServerAddress(),
+ loc.getStartCode(), loc.getHostName());
}
/**
@@ -712,7 +727,7 @@ public class ZooKeeperWrapper implements Watcher {
* @throws KeeperException
*/
public HServerInfo readAddressOrThrow(String znode, Watcher watcher)
- throws KeeperException {
+ throws KeeperException {
byte[] data;
try {
data = recoverableZK.getData(znode, watcher, null);
@@ -732,6 +747,54 @@ public class ZooKeeperWrapper implements Watcher {
}
}
+ public byte[] readRootZNode(String znode, Watcher watcher)
+ throws KeeperException {
+ byte[] data;
+ try {
+ data = recoverableZK.getData(znode, watcher, null);
+ } catch (InterruptedException e) {
+ // This should not happen
+ LOG.warn("Caught interrupted exception while trying to fetch "
+ + znode, e);
+ return null;
+ } catch (KeeperException.NoNodeException e) {
+ LOG.warn("Caught NoNode exception while trying to fetch "
+ + znode, e);
+ return null;
+ }
+ return data;
+ }
+
+ public RootRegionLocation readRootRegionLocationOrThrow(
+ String znode, Watcher watcher) throws KeeperException {
+ RootRegionLocation loc = null;
+ try {
+ byte[] data = readRootZNode(znode, watcher);
+ if (data == null) {
+ throw new RuntimeException("Obtained null data for znode : " + znode);
+ }
+ loc = Bytes.readThriftBytes(data,
+ RootRegionLocation.class);
+ } catch (Exception e) {
+ LOG.warn("Cannot deserialize RootRegionLocation", e);
+ throw new RuntimeException(e);
+ }
+ LOG.debug("<" + instanceName + ">" + "Read ZNode "
+ + znode + " got " + loc.toString());
+ return loc;
+ }
+
+ public RootRegionLocation readRootRegionLocation(String znode,
+ Watcher watcher) {
+ try {
+ LOG.debug("<" + instanceName + ">" + "Trying to read " + znode);
+ return readRootRegionLocationOrThrow(znode, watcher);
+ } catch (Exception e) {
+ LOG.debug("<" + instanceName + ">" + "Failed to read " + e.getMessage(), e);
+ return null;
+ }
+ }
+
/**
* Make sure this znode exists by creating it if it's missing
* @param znode full path to znode
@@ -841,8 +904,8 @@ public class ZooKeeperWrapper implements Watcher {
LOG.debug("<" + instanceName + ">" + "Deleted ZNode " + znode);
}
- private boolean createRootRegionLocation(String znode, String address) {
- byte[] data = Bytes.toBytes(address);
+ private boolean createRootRegionLocation(String znode,
+ byte[] data, String address) {
try {
recoverableZK.create(znode, data, Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
@@ -857,16 +920,19 @@ public class ZooKeeperWrapper implements Watcher {
return false;
}
- private boolean updateRootRegionLocation(String znode, String address) {
- byte[] data = Bytes.toBytes(address);
+ private boolean updateRootRegionLocation(String znode, byte[] data,
+ String dataPrettyStirng) {
try {
recoverableZK.setData(znode, data, -1);
- LOG.debug("<" + instanceName + ">" + "SetData of ZNode " + znode + " with " + address);
+ LOG.debug("<" + instanceName + ">" + "SetData of ZNode "
+ + znode + " with " + dataPrettyStirng);
return true;
} catch (KeeperException e) {
- LOG.warn("<" + instanceName + ">" + "Failed to set root region location in ZooKeeper: " + e);
+ LOG.warn("<" + instanceName + ">"
+ + "Failed to set root region location in ZooKeeper: " + e);
} catch (InterruptedException e) {
- LOG.warn("<" + instanceName + ">" + "Failed to set root region location in ZooKeeper: " + e);
+ LOG.warn("<" + instanceName + ">"
+ + "Failed to set root region location in ZooKeeper: " + e);
}
return false;
@@ -886,24 +952,46 @@ public class ZooKeeperWrapper implements Watcher {
return writeRootRegionLocation(hsi, true);
}
- private boolean writeRootRegionLocation(HServerInfo hsi, boolean isLegacyZNode) {
- String znode = isLegacyZNode ? legacyRootRegionZNode : rootRegionZNode;
- if (hsi == null) {
+ private boolean writeRootRegionLocation(String znode, byte[] data,
+ String dataString) {
+ if (data == null) {
return deleteRootRegionLocation(znode);
}
-
if (!ensureParentExists(znode)) {
return false;
}
-
- String addressString = isLegacyZNode ? hsi.getServerAddress().toString() :
- hsi.getServerName();
-
if (checkExistenceOf(znode)) {
- return updateRootRegionLocation(znode, addressString);
+ return updateRootRegionLocation(znode, data,
+ dataString);
+ }
+ return createRootRegionLocation(znode, data, dataString);
+ }
+
+ private boolean writeRootRegionLocation(HServerInfo hsi, boolean isLegacyZNode) {
+ String oldznode = isLegacyZNode ? legacyRootRegionZNode : rootRegionZNode;
+ if (hsi == null) {
+ return deleteRootRegionLocation(oldznode)
+ && deleteRootRegionLocation(rootRegionZNodeFinal);
}
+ String oldZnodeData = isLegacyZNode ?
+ hsi.getServerAddress().toString():hsi.getServerName();
- return createRootRegionLocation(znode, addressString);
+ RootRegionLocation znodeLocation =
+ new RootRegionLocation(hsi.getServerAddress().getHostname(),
+ hsi.getServerAddress().getPort(), hsi.getStartCode(),
+ HConstants.PROTOCOL_VERSION);
+
+ byte[] znodeData;
+ try {
+ znodeData =
+ Bytes.writeThriftBytes(znodeLocation, RootRegionLocation.class);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return writeRootRegionLocation(oldznode,
+ Bytes.toBytes(oldZnodeData), oldZnodeData)
+ && writeRootRegionLocation(rootRegionZNodeFinal,
+ znodeData, znodeLocation.toString());
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/e14e5d54/src/test/java/org/apache/hadoop/hbase/client/TestRootRegionLocationZNode.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRootRegionLocationZNode.java b/src/test/java/org/apache/hadoop/hbase/client/TestRootRegionLocationZNode.java
new file mode 100644
index 0000000..53e4898
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRootRegionLocationZNode.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestRootRegionLocationZNode {
+ private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @Before
+ public void setUp() throws Exception {
+ UTIL.startMiniCluster();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRootRegionLocation()
+ throws IOException, KeeperException, InterruptedException {
+ TableServers conn =
+ (TableServers)HConnectionManager.getConnection(UTIL.getConfiguration());
+ ZooKeeperWrapper zk = conn.getZooKeeperWrapper();
+ HRegionLocation loc = conn.locateRootRegion();
+ zk.deleteZNode("/hbase/root-region-server-final");
+ HRegionLocation loc2 = conn.locateRootRegion();
+ assertTrue(loc.getServerAddress().equals(loc2.getServerAddress()));
+ }
+}
[50/50] [abbrv] git commit: [master] Upgrade jetty version to 6.1.26
from 6.1.24
Posted by ec...@apache.org.
[master] Upgrade jetty version to 6.1.26 from 6.1.24
Summary: David found some runtime version conflict in Jetty. This diff is to fix the same.
Test Plan: HBase builds without any problems.
Reviewers: daviddeng, rshroff
Reviewed By: rshroff
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1433390
Tasks: 4656190
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43455 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a12267f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a12267f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a12267f6
Branch: refs/heads/0.89-fb
Commit: a12267f6c64eff540c0d6e2a111aee2190f54e18
Parents: da748d4
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jul 14 18:31:18 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 15:01:33 2014 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a12267f6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 02e8ea3..7c509aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -663,7 +663,7 @@
<commons-cli.version>1.2</commons-cli.version>
<commons-io.version>2.1</commons-io.version>
<commons-logging.version>1.1.1</commons-logging.version>
- <jetty.version>6.1.24</jetty.version>
+ <jetty.version>6.1.26</jetty.version>
<jetty.jspapi.version>6.1.14</jetty.jspapi.version>
<netty.version>3.7.0.Final</netty.version>
<junit.version>4.11</junit.version>
[48/50] [abbrv] git commit: [HBASE-11636] Bring back test annotations.
Posted by ec...@apache.org.
[HBASE-11636] Bring back test annotations.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5808b33d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5808b33d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5808b33d
Branch: refs/heads/0.89-fb
Commit: 5808b33d089ce2aa01ba6e708fbbdea24637b885
Parents: cfa65f8
Author: Elliott Clark <el...@fb.com>
Authored: Thu Jul 31 14:48:50 2014 -0700
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:48:50 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/LargeTests.java | 38 ++++++++++++++++++++
.../org/apache/hadoop/hbase/MediumTests.java | 36 +++++++++++++++++++
.../org/apache/hadoop/hbase/SmallTests.java | 33 +++++++++++++++++
.../org/apache/hadoop/hbase/UnstableTests.java | 7 ++++
4 files changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5808b33d/src/main/java/org/apache/hadoop/hbase/LargeTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/LargeTests.java b/src/main/java/org/apache/hadoop/hbase/LargeTests.java
new file mode 100644
index 0000000..958ffd7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/LargeTests.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+/**
+ * Tag a test as 'large', meaning that the test class has the following
+ * characteristics:
+ * - executed in an isolated JVM. Tests can however be executed in different
+ * JVM on the same machine simultaneously.
+ * - will not have to be executed by the developer before submitting a bug
+ * - ideally, last less than 2 minutes to help parallelization
+ *
+ * It the worst case compared to small or medium, use it only for tests that
+ * you cannot put in the other categories
+ *
+ * @see SmallTests
+ * @see MediumTests
+ * @see IntegrationTests
+ */
+public interface LargeTests {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5808b33d/src/main/java/org/apache/hadoop/hbase/MediumTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/MediumTests.java b/src/main/java/org/apache/hadoop/hbase/MediumTests.java
new file mode 100644
index 0000000..e99f292
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/MediumTests.java
@@ -0,0 +1,36 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+/**
+ * Tag a test as 'Medium', meaning that the test class has the following
+ * characteristics:
+ * - executed in an isolated JVM. Tests can however be executed in different
+ * JVM on the same machine simultaneously.
+ * - will have to be executed by the developer before submitting a bug
+ * - ideally, last less than 1 minutes to help parallelization
+ *
+ * Use it for tests that cannot be tagged as 'Small'.
+ *
+ * @see SmallTests
+ * @see LargeTests
+ */
+public interface MediumTests {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5808b33d/src/main/java/org/apache/hadoop/hbase/SmallTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/SmallTests.java b/src/main/java/org/apache/hadoop/hbase/SmallTests.java
new file mode 100644
index 0000000..d68a118
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/SmallTests.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+/**
+ * Tag a test as 'small', meaning that the test class has the following
+ * characteristics:
+ * - can be run simultaneously with other small tests in the same JVM
+ * - ideally, last less than 15 seconds
+ * - does not use a cluster
+ *
+ * @see MediumTests
+ * @see LargeTests
+ */
+public interface SmallTests {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5808b33d/src/main/java/org/apache/hadoop/hbase/UnstableTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/UnstableTests.java b/src/main/java/org/apache/hadoop/hbase/UnstableTests.java
new file mode 100644
index 0000000..471ab7c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/UnstableTests.java
@@ -0,0 +1,7 @@
+package org.apache.hadoop.hbase;
+
+/**
+ * Test that is unstable. This will not be run.
+ */
+public interface UnstableTests {
+}
[30/50] [abbrv] [HBASE-11379] Modified AlterTable to fix stale
.regioninfo and wrote tests
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/test/java/org/apache/hadoop/hbase/master/TestAlterTableRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAlterTableRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/master/TestAlterTableRegionInfo.java
new file mode 100644
index 0000000..f2fc120
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAlterTableRegionInfo.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hbase.master;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Category(MediumTests.class)
+public class TestAlterTableRegionInfo {
+ private final Log LOG = LogFactory.getLog(TestAlterTableLocking.class);
+ private final static HBaseTestingUtility testingUtility = new HBaseTestingUtility();
+ private static final byte[] TABLENAME = Bytes.toBytes("TestTable");
+ private static final byte[] FAMILY1NAME = Bytes.toBytes("cf1");
+ private static final byte[] FAMILY2NAME = Bytes.toBytes("cf2");
+ private static final int NUM_REGIONS = 20;
+ private static final int NUM_VERSIONS = 1;
+ private static final byte[] START_KEY = Bytes.toBytes(0);
+ private static final byte[] END_KEY = Bytes.toBytes(200);
+ private static final int WAIT_INTERVAL = 1000;
+ private static final int MAX_CONCURRENT_REGIONS_CLOSED = 2;
+ private static final int WAIT_FOR_TABLE = 30 * 1000;
+
+ @BeforeClass
+ public static void setup() throws IOException, InterruptedException {
+ //Start mini cluster
+ testingUtility.startMiniCluster(3);
+ testingUtility.createTable(
+ TABLENAME,
+ new byte[][]{ FAMILY1NAME, FAMILY2NAME },
+ NUM_VERSIONS,
+ START_KEY,
+ END_KEY,
+ NUM_REGIONS
+ );
+ testingUtility.waitTableAvailable(TABLENAME, WAIT_FOR_TABLE);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ testingUtility.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRegionInfoConsistency() throws IOException {
+ HBaseAdmin admin = null;
+ admin = testingUtility.getHBaseAdmin();
+
+ List<HColumnDescriptor> columnAdditions = new ArrayList<>();
+ columnAdditions.add(new HColumnDescriptor("cf3"));
+
+ List<Pair<byte[], HColumnDescriptor>> columnModifications = new ArrayList<>();
+ columnModifications.add(new Pair<byte[], HColumnDescriptor>(Bytes.toBytes("cf1"),
+ new HColumnDescriptor("cf1modified")));
+
+ List<byte[]> columnDeletions = new ArrayList<>();
+ columnDeletions.add(Bytes.toBytes("cf2"));
+
+ admin.alterTable(TABLENAME, columnAdditions, columnModifications, columnDeletions,
+ WAIT_INTERVAL, MAX_CONCURRENT_REGIONS_CLOSED);
+ Set<byte[]> families = admin.getTableDescriptor(TABLENAME).getFamiliesKeys();
+
+ // each entry in allReadFamilies contains the set of families of a particular .regioninfo file
+ List<Set<byte[]>> allReadFamilies = readAllRegionInfoFamilies(admin, TABLENAME);
+ for (Set<byte[]> readFamilies : allReadFamilies) {
+ // each set of families must be correct
+ Assert.assertEquals("Expected was " + families + ", got " + readFamilies + ".", families,
+ readFamilies);
+ }
+ }
+
+ /**
+ * This methods reads all the .regioninfo file of every region in a particular table and finds the
+ * set of families in each file
+ * @param admin
+ * @param tableName -- name of table to be examined
+ * @return List of Sets. Each Set contains the column family names of a particular region
+ * @throws IOException
+ */
+ List<Set<byte[]>> readAllRegionInfoFamilies(HBaseAdmin admin, byte[] tableName)
+ throws IOException {
+ List<Set<byte[]>> allReadFamilies = new ArrayList<Set<byte[]>>();
+ Configuration conf = admin.getConnection().getConf();
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ HTable table = new HTable(conf, tableName);
+ byte[] tableDescName = table.getTableDescriptor().getName();
+
+ // iterate through all of the table's regions and read their .regioninfo files
+ for (HRegionInfo hri : table.getRegionsInfo().keySet()) {
+ Path tableDir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), tableDescName);
+ Path regionPath = HRegion.getRegionDir(tableDir, hri.getEncodedName());
+ Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
+
+ FSDataInputStream in = fs.open(regionInfoPath);
+ HRegionInfo f_hri = new HRegionInfo();
+ try {
+ f_hri.readFields(in);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ in.close();
+ }
+
+ allReadFamilies.add(f_hri.getTableDesc().getFamiliesKeys());
+ }
+ table.close();
+ return allReadFamilies;
+ }
+
+
+}
[23/50] [abbrv] git commit: [HBASE-11379] added fixRegionInfo method
to rewrite stale .regioninfo files
Posted by ec...@apache.org.
[HBASE-11379] added fixRegionInfo method to rewrite stale .regioninfo files
Summary: added method in HBaseFsck to rewrite .regioninfo files that are inconsistent with .META
Test Plan: intentionally corrupted .regioninfo files and then fixed them
Reviewers: manukranthk
Reviewed By: manukranthk
Subscribers: adela, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1389834
Tasks: 4490604
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42987 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d8ba4507
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d8ba4507
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d8ba4507
Branch: refs/heads/0.89-fb
Commit: d8ba4507b6ad798f049432c790ff8945b7256d3c
Parents: cd54a01
Author: yuq <yu...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 19 19:24:16 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/client/HBaseFsck.java | 169 ++++++++++++------
.../client/TestHBaseFsckFixRegionInfo.java | 178 +++++++++++++++++++
2 files changed, 288 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8ba4507/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
index cab5e01..5206f4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
@@ -46,6 +46,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -79,7 +80,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
/**
- * Check consistency among the in-memory states of the master and the
+ * Check and enforce consistency among the in-memory states of the master and the
* region server(s) and the state of data in HDFS.
*/
public class HBaseFsck {
@@ -104,6 +105,7 @@ public class HBaseFsck {
private boolean rerun = false; // if we tried to fix something rerun hbck
private static boolean summary = false; // if we want to print less output
private static boolean checkRegionInfo = false;
+ private static boolean fixRegionInfo = false;
private static boolean promptResponse = false; // "no" to all prompt questions
private int numThreads = MAX_NUM_THREADS;
private final String UnkownTable = "__UnknownTable__";
@@ -128,7 +130,7 @@ public class HBaseFsck {
* @throws MasterNotRunningException if the master is not running
*/
public HBaseFsck(Configuration conf)
- throws MasterNotRunningException, IOException {
+ throws MasterNotRunningException, IOException {
this.conf = conf;
// fetch information from master
@@ -138,8 +140,8 @@ public class HBaseFsck {
numThreads = conf.getInt("hbasefsck.numthreads", numThreads);
executor = new ThreadPoolExecutor(numThreads, numThreads,
- THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>());
+ THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
executor.allowCoreThreadTimeOut(true);
}
@@ -195,17 +197,17 @@ public class HBaseFsck {
for (HTableDescriptor td : allTables) {
String tableName = td.getNameAsString();
errors.detail("\t Table: " + tableName + "\t" +
- (td.isReadOnly() ? "ro" : "rw") + "\t" +
- (td.isRootRegion() ? "ROOT" :
- (td.isMetaRegion() ? "META" : " ")) + "\t" +
- " families:" + td.getFamilies().size());
+ (td.isReadOnly() ? "ro" : "rw") + "\t" +
+ (td.isRootRegion() ? "ROOT" :
+ (td.isMetaRegion() ? "META" : " ")) + "\t" +
+ " families:" + td.getFamilies().size());
}
}
// From the master, get a list of all known live region servers
Collection<HServerInfo> regionServers = status.getServerInfo();
errors.print("Number of live region servers:" +
- regionServers.size());
+ regionServers.size());
if (details) {
for (HServerInfo rsinfo: regionServers) {
errors.detail("\t RegionServer:" + rsinfo.getServerName());
@@ -215,7 +217,7 @@ public class HBaseFsck {
// From the master, get a list of all dead region servers
Collection<String> deadRegionServers = status.getDeadServerNames();
errors.print("Number of dead region servers:" +
- deadRegionServers.size());
+ deadRegionServers.size());
if (details) {
for (String name: deadRegionServers) {
errors.detail("\t RegionServer(dead):" + name);
@@ -242,6 +244,11 @@ public class HBaseFsck {
checkRegionInfo();
}
+ //Rewrite .regioninfo files that are inconsistent with .META
+ if (fixRegionInfo){
+ fixRegionInfo();
+ }
+
// Print table summary
printTableSummary();
@@ -252,9 +259,11 @@ public class HBaseFsck {
* Read the .regioninfo for all regions of the table and compare with the
* corresponding entry in .META.
* Entry in .regioninfo should be consistent with the entry in .META.
- *
+ * @return Map<HRegionInfo, Path> containing the HRegionInfo objects corresponding to
+ * HRegions whose .regioninfo files should be rewritten and the Paths to be written at
*/
- void checkRegionInfo() {
+ Map<HRegionInfo, Path> checkRegionInfo() {
+ Map<HRegionInfo, Path> risToRewrite = new TreeMap<HRegionInfo, Path>();
Path tableDir = null;
try {
@@ -266,10 +275,11 @@ public class HBaseFsck {
FileSystem fs = rootDir.getFileSystem(conf);
Path regionPath = HRegion.getRegionDir(tableDir,
- hbi.metaEntry.getEncodedName());
+ hbi.metaEntry.getEncodedName());
Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
+
if (fs.exists(regionInfoPath) &&
- fs.getFileStatus(regionInfoPath).getLen() > 0) {
+ fs.getFileStatus(regionInfoPath).getLen() > 0) {
FSDataInputStream in = fs.open(regionInfoPath);
HRegionInfo f_hri = null;
try {
@@ -277,7 +287,7 @@ public class HBaseFsck {
f_hri.readFields(in);
} catch (IOException ex) {
errors.reportError("Could not read .regioninfo file at "
- + regionInfoPath);
+ + regionInfoPath);
} finally {
in.close();
}
@@ -285,23 +295,60 @@ public class HBaseFsck {
HRegionInfo m_hri = hbckinfo.metaEntry;
if(!f_hri.equals(m_hri)) {
errors.reportError("Table name: " +
- f_hri.getTableDesc().getNameAsString() +
- " RegionInfo for "+ f_hri.getRegionNameAsString() +
- " inconsistent in .META. and .regioninfo");
+ f_hri.getTableDesc().getNameAsString() +
+ " RegionInfo for "+ f_hri.getRegionNameAsString() +
+ " inconsistent in .META. and .regioninfo");
+ risToRewrite.put(m_hri, regionInfoPath);
}
} else {
if (!fs.exists(regionInfoPath)) {
errors.reportError(".regioninfo not found at "
- + regionInfoPath.toString());
+ + regionInfoPath.toString());
} else if (fs.getFileStatus(regionInfoPath).getLen() <= 0) {
errors.reportError(".regioninfo file is empty (path = "
- + regionInfoPath + ")");
+ + regionInfoPath + ")");
}
}
}
} catch (IOException e) {
errors.reportError("Error in comparing .regioninfo and .META."
- + e.getMessage());
+ + e.getMessage());
+ }
+
+ return risToRewrite;
+ }
+
+ /**
+ * Read the .regioninfo for all regions of the table and compare with the
+ * corresponding entry in .META.
+ * Entry in .regioninfo should be consistent with the entry in .META.
+ * If .regioninfo is not consistent with .META, then we rewrite the .regioninfo
+ * @throws IOException
+ */
+ void fixRegionInfo() throws IOException {
+ Map<HRegionInfo, Path> risToRewrite = checkRegionInfo();
+ LOG.info("FOUND " + risToRewrite.size() + " corrupted .regioninfo files to rewrite");
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ for(Map.Entry<HRegionInfo, Path> entry : risToRewrite.entrySet()){
+ HRegionInfo hri = entry.getKey();
+ Path regionInfoPath = entry.getValue();
+ FSDataOutputStream out = fs.create(regionInfoPath, true);
+
+ try {
+ hri.write(out);
+ out.write('\n');
+ out.write('\n');
+ out.write(Bytes.toBytes(hri.toString()));
+ LOG.debug("Rewrote .regioninfo file of " + Bytes.toString(hri.getRegionName()) +"at " + regionInfoPath);
+ } catch (IOException e){
+ errors.reportError("Could not rewrite the .regioninfo of "+Bytes.toString(hri.getRegionName()) + " at " +regionInfoPath);
+ }
+
+ finally {
+ out.close();
+ LOG.debug(".regioninfo File Contents: " + Bytes.toBytes(hri.toString()));
+ }
}
}
@@ -347,7 +394,7 @@ public class HBaseFsck {
*/
boolean recordRootRegion() throws IOException {
HRegionLocation rootLocation = connection.locateRegion(
- HConstants.ROOT_TABLE_NAME_STRINGBYTES, HConstants.EMPTY_START_ROW);
+ HConstants.ROOT_TABLE_NAME_STRINGBYTES, HConstants.EMPTY_START_ROW);
// Check if Root region is valid and existing
if (rootLocation == null || rootLocation.getRegionInfo() == null ||
@@ -357,7 +404,7 @@ public class HBaseFsck {
}
MetaEntry m = new MetaEntry(rootLocation.getRegionInfo(),
- rootLocation.getServerAddress(), null, System.currentTimeMillis());
+ rootLocation.getServerAddress(), null, System.currentTimeMillis());
HbckInfo hbInfo = new HbckInfo(m);
regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo);
return true;
@@ -430,13 +477,13 @@ public class HBaseFsck {
boolean isDeployed = !hbi.deployedOn.isEmpty();
boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
boolean deploymentMatchesMeta =
- hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
- hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
+ hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
+ hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
boolean shouldBeDeployed = inMeta && !hbi.metaEntry.isOffline();
long tooRecent = System.currentTimeMillis() - timelag;
boolean recentlyModified =
- (inHdfs && hbi.foundRegionDir.getModificationTime() > tooRecent) ||
- (inMeta && hbi.metaEntry.modTime > tooRecent);
+ (inHdfs && hbi.foundRegionDir.getModificationTime() > tooRecent) ||
+ (inMeta && hbi.metaEntry.modTime > tooRecent);
String tableName = null;
if (inMeta) {
@@ -478,23 +525,23 @@ public class HBaseFsck {
assert false : "Entry for region with no data";
} else if (!inMeta && !inHdfs && isDeployed) {
String message = "Region " + descriptiveName + " not on HDFS or in META but " +
- "deployed on " + Joiner.on(", ").join(hbi.deployedOn);
+ "deployed on " + Joiner.on(", ").join(hbi.deployedOn);
errors.reportError(message);
tableInfo.addRegionDetails(RegionType.missing, descriptiveName);
tableInfo.addRegionError(message);
} else if (!inMeta && inHdfs && !isDeployed) {
String message = "Region " + descriptiveName + " on HDFS, but not listed in META " +
- "or deployed on any region server.";
+ "or deployed on any region server.";
errors.reportError(message);
tableInfo.addRegionDetails(RegionType.missing, descriptiveName);
tableInfo.addRegionError(message);
} else if (!inMeta && inHdfs && isDeployed) {
String message = "Region " + descriptiveName + " not in META, but deployed on " +
- Joiner.on(", ").join(hbi.deployedOn);
+ Joiner.on(", ").join(hbi.deployedOn);
errors.reportError(message);
tableInfo.addRegionDetails(RegionType.missing, descriptiveName);
tableInfo.addRegionError(message);
- // ========== Cases where the region is in META =============
+ // ========== Cases where the region is in META =============
} else if (inMeta && !inHdfs && !isDeployed) {
String message = "Region " + descriptiveName + " found in META, but not in HDFS " +
"or deployed on any region server.";
@@ -503,7 +550,7 @@ public class HBaseFsck {
tableInfo.addRegionError(message);
} else if (inMeta && !inHdfs && isDeployed) {
String message = "Region " + descriptiveName + " found in META, but not in HDFS, " +
- "and deployed on " + Joiner.on(", ").join(hbi.deployedOn);
+ "and deployed on " + Joiner.on(", ").join(hbi.deployedOn);
errors.reportError(message);
tableInfo.addRegionDetails(RegionType.unknown, descriptiveName);
tableInfo.addRegionError(message);
@@ -720,7 +767,7 @@ public class HBaseFsck {
if (Bytes.compareTo(prevEndKey, curStartKey) != 0) {
this.holesFound++;
errorSB.append("Found hole in table from start key "
- + posToStr(prevEndKey) + " to end key " + posToStr(curStartKey) + "\n");
+ + posToStr(prevEndKey) + " to end key " + posToStr(curStartKey) + "\n");
}
visited.add(curStartKey);
@@ -730,8 +777,8 @@ public class HBaseFsck {
if (visited.contains(prevEndKey)
&& Bytes.compareTo(prevEndKey, new byte[0]) != 0) {
errorSB.append("Cycle found in region chain. "
- + "Current Start Key "+ posToStr(curStartKey)
- + "; Cycle Start Key " + posToStr(prevEndKey) + "\n");
+ + "Current Start Key "+ posToStr(curStartKey)
+ + "; Cycle Start Key " + posToStr(prevEndKey) + "\n");
}
}
@@ -739,7 +786,7 @@ public class HBaseFsck {
if (Bytes.compareTo(prevEndKey, new byte[0]) != 0) {
this.holesFound++;
errorSB.append("Found hole in table from start key "
- + posToStr(prevEndKey) + " to end of table (end key 0)\n");
+ + posToStr(prevEndKey) + " to end of table (end key 0)\n");
}
if (this.holesFound > 0) {
@@ -854,10 +901,10 @@ public class HBaseFsck {
}
/**
- * Get values in regionInfo for .META.
- *
- * @return list of meta regions
- */
+ * Get values in regionInfo for .META.
+ *
+ * @return list of meta regions
+ */
private List<HbckInfo> getMetaRegions() {
List<HbckInfo> metaRegions = Lists.newArrayList();
for (HbckInfo value : regionInfo.values()) {
@@ -894,7 +941,7 @@ public class HBaseFsck {
// record region details
byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
+ HConstants.REGIONINFO_QUALIFIER);
HRegionInfo info = null;
HServerAddress server = null;
byte[] startCode = null;
@@ -904,7 +951,7 @@ public class HBaseFsck {
// record assigned region server
value = result.getValue(HConstants.CATALOG_FAMILY,
- HConstants.SERVER_QUALIFIER);
+ HConstants.SERVER_QUALIFIER);
if (value != null && value.length > 0) {
String address = Bytes.toString(value);
server = new HServerAddress(address);
@@ -912,7 +959,7 @@ public class HBaseFsck {
// record region's start key
value = result.getValue(HConstants.CATALOG_FAMILY,
- HConstants.STARTCODE_QUALIFIER);
+ HConstants.STARTCODE_QUALIFIER);
if (value != null) {
startCode = value;
}
@@ -999,7 +1046,7 @@ public class HBaseFsck {
long modTime; // timestamp of most recent modification metadata
public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer,
- byte[] startCode, long modTime) {
+ byte[] startCode, long modTime) {
super(rinfo);
this.regionServer = regionServer;
this.modTime = modTime;
@@ -1108,9 +1155,9 @@ public class HBaseFsck {
public synchronized int summarize() {
if (HBaseFsck.json == null) {
System.out.println(Integer.toString(errorCount + fixableCount) +
- " inconsistencies detected.");
+ " inconsistencies detected.");
System.out.println(Integer.toString(fixableCount) +
- " inconsistencies are fixable.");
+ " inconsistencies are fixable.");
}
if (warnCount > 0) {
if (HBaseFsck.json == null) {
@@ -1181,7 +1228,7 @@ public class HBaseFsck {
private boolean done;
WorkItemRegion(HBaseFsck hbck, HServerInfo info,
- ErrorReporter errors, HConnection connection) {
+ ErrorReporter errors, HConnection connection) {
this.hbck = hbck;
this.rsinfo = info;
this.errors = errors;
@@ -1200,7 +1247,7 @@ public class HBaseFsck {
errors.progress();
try {
HRegionInterface server = connection.getHRegionConnection(
- rsinfo.getServerAddress());
+ rsinfo.getServerAddress());
// list all online regions from this region server
HRegionInfo[] regions = server.getRegionsAssignment();
@@ -1208,13 +1255,13 @@ public class HBaseFsck {
if (details) {
StringBuffer buf = new StringBuffer();
buf.append("\nRegionServer:" + rsinfo.getServerName() +
- " number of regions:" + regions.length);
+ " number of regions:" + regions.length);
for (HRegionInfo rinfo: regions) {
buf.append("\n\t name:" + rinfo.getRegionNameAsString() +
- " id:" + rinfo.getRegionId() +
- " encoded name:" + rinfo.getEncodedName() +
- " start :" + Bytes.toStringBinary(rinfo.getStartKey()) +
- " end :" + Bytes.toStringBinary(rinfo.getEndKey()));
+ " id:" + rinfo.getRegionId() +
+ " encoded name:" + rinfo.getEncodedName() +
+ " start :" + Bytes.toStringBinary(rinfo.getStartKey()) +
+ " end :" + Bytes.toStringBinary(rinfo.getEndKey()));
}
errors.detail(buf.toString());
}
@@ -1226,7 +1273,7 @@ public class HBaseFsck {
}
} catch (IOException e) { // unable to connect to the region server.
errors.reportWarning("RegionServer: " + rsinfo.getServerName()
- + " Unable to fetch region information. " + e);
+ + " Unable to fetch region information. " + e);
hbck.addFailedServer(rsinfo.getServerAddress());
} finally {
done = true;
@@ -1246,7 +1293,7 @@ public class HBaseFsck {
private boolean done;
WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
- FileStatus status) {
+ FileStatus status) {
this.hbck = hbck;
this.fs = fs;
this.tableDir = status;
@@ -1279,7 +1326,7 @@ public class HBaseFsck {
synchronized (hbi) {
if (hbi.foundRegionDir != null) {
errors.print("Directory " + encodedName + " duplicate??" +
- hbi.foundRegionDir);
+ hbi.foundRegionDir);
}
hbi.foundRegionDir = regionDir;
@@ -1299,7 +1346,7 @@ public class HBaseFsck {
}
} catch (IOException e) { // unable to connect to the region server.
errors.reportError("Table Directory: " + tableDir.getPath().getName() +
- " Unable to fetch region information. " + e);
+ " Unable to fetch region information. " + e);
} finally {
done = true;
notifyAll();
@@ -1395,7 +1442,7 @@ public class HBaseFsck {
* @throws ParseException
*/
public static void main(String [] args)
- throws IOException,
+ throws IOException,
MasterNotRunningException, InterruptedException, ParseException {
Options opt = new Options();
@@ -1422,6 +1469,7 @@ public class HBaseFsck {
opt.addOption("summary", false, "Print only summary of the tables and status.");
opt.addOption("detail", false, "Display full report of all regions.");
opt.addOption("checkRegionInfo", false, "Check if .regioninfo is consistent with .META.");
+ opt.addOption("fixRegionInfo", false, "Rewrite all .regioninfo that are inconsistent with .META");
opt.addOption("h", false, "Display this help");
opt.addOption("json", false, "Outputs the table summaries and errors in JSON format");
opt.addOption("forceCleanMeta", false,
@@ -1492,11 +1540,14 @@ public class HBaseFsck {
HBaseFsck.setPromptResponse(true);
}
if (cmd.hasOption("summary")) {
- fsck.setSummary();
+ fsck.setSummary();
}
if (cmd.hasOption("checkRegionInfo")) {
checkRegionInfo = true;
}
+ if (cmd.hasOption("fixRegionInfo")){
+ fixRegionInfo = true;
+ }
if (cmd.hasOption("json")) {
fsck.setJsonFlag("json");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8ba4507/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
new file mode 100644
index 0000000..44738aa
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
@@ -0,0 +1,178 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Class to test the checkRegionInfo() and fixRegionInfo() methods of HBaseFsck.
+ */
+@Category(MediumTests.class)
+public class TestHBaseFsckFixRegionInfo {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private HBaseAdmin admin;
+ private static final int NUM_REGION_SERVER = 3;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+ TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+ TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+ TEST_UTIL.startMiniCluster(NUM_REGION_SERVER);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ }
+
+ @Test
+ public void testFixRegionInfo() throws IOException, InterruptedException {
+
+ byte[] tableName = Bytes.toBytes("testFixRegionInfo");
+
+ byte[][] splitKeys = {
+ new byte[] { 1, 1, 1 },
+ new byte[] { 2, 2, 2 },
+ new byte[] { 3, 3, 3 },
+ new byte[] { 4, 4, 4 },
+ new byte[] { 5, 5, 5 },
+ new byte[] { 6, 6, 6 },
+ new byte[] { 7, 7, 7 },
+ new byte[] { 8, 8, 8 },
+ new byte[] { 9, 9, 9 },
+ };
+
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+ admin.createTable(desc, splitKeys);
+
+ Configuration conf = TEST_UTIL.getConfiguration();
+ HBaseFsck fsckToCorrupt = new HBaseFsck(TEST_UTIL.getConfiguration());
+ fsckToCorrupt.initAndScanRootMeta();
+ /*
+ * The following line is necessary because some .regioninfo files (specifically the ROOT and the META)
+ * become outdated immediately after initialization. This is because some parameters are immediately changed
+ * after initialization, and these changes are not updated in the .regioninfo. So here we update all files.
+ * This allows us to accurate measure the # of corrupted files later.
+ */
+ fsckToCorrupt.fixRegionInfo();
+
+ //Randomly corrupt some files
+ int modifyCount = 0; // # of corrupted files
+ for (HbckInfo hbi : fsckToCorrupt.getRegionInfo().values()) {
+ if (Math.random() < 0.5) {
+ Path tableDir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
+ hbi.metaEntry.getTableDesc().getName());
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ Path regionPath = HRegion.getRegionDir(tableDir,
+ hbi.metaEntry.getEncodedName());
+ Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
+
+ //read the original .regioninfo into an HRegionInfo
+ FSDataInputStream in = fs.open(regionInfoPath);
+ HRegionInfo hri_original = new HRegionInfo();
+ hri_original.readFields(in);
+ in.close();
+
+ //change the HRegionInfo
+ HRegionInfo hri_modified = randomlyModifyRegion(hri_original);
+
+ //rewrite the original .regioninfo
+ FSDataOutputStream out = fs.create(regionInfoPath, true);
+ hri_modified.write(out);
+ out.write('\n');
+ out.write('\n');
+ out.write(Bytes.toBytes(hri_modified.toString()));
+ out.close();
+ modifyCount++;
+ }
+ }
+
+ //we incorrectly rewrote some .regioninfo files, so some .regioninfos are incorrect
+ HBaseFsck fsckCorrupted = new HBaseFsck(TEST_UTIL.getConfiguration());
+ fsckCorrupted.initAndScanRootMeta();
+ Map<HRegionInfo, Path> risToRewrite = fsckCorrupted.checkRegionInfo();
+ assertEquals(modifyCount, risToRewrite.size()); // # of files to rewrite must be the # of modified files
+
+ //after fixing, we should see no errors
+ HBaseFsck fsckFixed = new HBaseFsck(TEST_UTIL.getConfiguration());
+ fsckFixed.initAndScanRootMeta();
+ fsckFixed.fixRegionInfo();
+ risToRewrite = fsckFixed.checkRegionInfo();
+ assertEquals(0, risToRewrite.size());
+ }
+
+ HRegionInfo randomlyModifyRegion(HRegionInfo hri){
+ HTableDescriptor tableDesc = hri.getTableDesc();
+ double rand = Math.random();
+ if (rand < 0.2){
+ tableDesc.setReadOnly(!tableDesc.isReadOnly());
+ }
+ else if (rand < 0.4){
+ tableDesc.setDeferredLogFlush(!tableDesc.isDeferredLogFlush());
+ }
+ else if (rand < 0.6){
+ tableDesc.setName(Bytes.toBytes("NEWNAME"));
+ }
+ else if (rand < 0.8){
+ tableDesc.setMaxFileSize( (long) 100);
+ }
+ else{
+ tableDesc.setMemStoreFlushSize( (long) 200);
+ }
+ hri.setTableDesc(tableDesc);
+ return hri;
+ }
+
+}
[44/50] [abbrv] git commit: [HBASE-11592] Allow number of
SplitLogWorker threads to be online-configurable
Posted by ec...@apache.org.
[HBASE-11592] Allow number of SplitLogWorker threads to be online-configurable
Test Plan:
mvn test -Dtest.output=true -Dtest=TestRegionServerOnlineConfigChange --
successful
Reviewers: avandever, manukranthk, rshroff, aaiyer, adela, fan, daviddeng, gauravm
Reviewed By: gauravm
CC: hbase-eng@fb.com
Differential Revision: https://phabricator.fb.com/D1461935
Tasks: 4516873
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43780 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfa65f89
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfa65f89
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfa65f89
Branch: refs/heads/0.89-fb
Commit: cfa65f89c6af2b66890a4e651c5b1b84bc772e0e
Parents: 7281aa1
Author: jgoguen <jg...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jul 30 17:00:28 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 5 ++
.../hbase/regionserver/HRegionServer.java | 75 +++++++++++++++++---
.../TestRegionServerOnlineConfigChange.java | 21 +++++-
3 files changed, 90 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfa65f89/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 8be87a4..24cc8f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -397,6 +397,11 @@ public final class HConstants {
"hbase.hregionserver.hlog.split.workers.num";
/**
+ * The default number of threads used for splitting logs in parallel.
+ */
+ public static final int DEFAULT_HREGIONSERVER_SPLITLOG_WORKERS_NUM = 3;
+
+ /**
* If using quorum reads from HDFS, the maximum size of the thread pool.
* value <= 0 disables quorum reads.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfa65f89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7c1c736..207760f 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1976,16 +1976,8 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
this.server.start();
isRpcServerRunning = true;
}
- int numSplitLogWorkers = conf.getInt(HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, 3);
- // Create the log splitting worker and start it
- this.splitLogWorkers = new ArrayList<SplitLogWorker>(numSplitLogWorkers);
- for (int i = 0; i < numSplitLogWorkers; i++) {
- SplitLogWorker splitLogWorker = new SplitLogWorker(this.zooKeeperWrapper,
- this.getConfiguration(), this.serverInfo.getServerName(),
- logCloseThreadPool, masterRef);
- this.splitLogWorkers.add(splitLogWorker);
- splitLogWorker.start();
- }
+ int numSplitLogWorkers = conf.getInt(HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, HConstants.DEFAULT_HREGIONSERVER_SPLITLOG_WORKERS_NUM);
+ this.setNumberOfSplitLogWorkerThreads(numSplitLogWorkers);
// start the scanner prefetch threadpool
int numHandlers = conf.getInt("hbase.regionserver.handler.count", 10);
scanPrefetchThreadPool =
@@ -1996,6 +1988,62 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
this.serverInfo.getServerAddress().toString());
}
+ /**
+ * Set the number of {@link SplitLogWorker} threads available. This is
+ * intended to be called at any time during the HRegionServer's lifetime,
+ * first called during {@link #startServiceThreads()}, and will create the
+ * backing {@link ArrayList} if it doesn't already exist.
+ *
+ * When the number of worker threads to use is less than the number of
+ * workers in use, extraneous workers are stopped. Worker threads at the
+ * start of the {@link ArrayList} are left alone, threads beyond index
+ * (numWorkers - 1) are stopped.
+ *
+ * When the number of worker threads to use is greater than the number of
+ * workers in use, the appropriate number of additional {@link SplitLogWorker}
+ * threads are added.
+ *
+ * @param numWorkers The number of worker threads to have running.
+ */
+ private void setNumberOfSplitLogWorkerThreads(int numWorkers) {
+ if (numWorkers <= 0 || (this.splitLogWorkers != null && numWorkers == this.splitLogWorkers.size())) {
+ // Don't accept zero or fewer workers, and don't do anything if we try to
+ // set the same number of threads already in use.
+ return;
+ }
+
+ if (null == this.splitLogWorkers) {
+ this.splitLogWorkers = new ArrayList<SplitLogWorker>(numWorkers);
+ }
+
+ if (numWorkers < this.splitLogWorkers.size()) {
+ // Fewer workers, stop threads
+ for (int idx = this.splitLogWorkers.size() - 1; idx >= numWorkers; idx--) {
+ this.splitLogWorkers.remove(idx).stop();
+ }
+ } else {
+ // More workers, spin up threads
+ int workersNeeded = numWorkers - this.splitLogWorkers.size();
+ do {
+ SplitLogWorker splitLogWorker = new SplitLogWorker(this.zooKeeperWrapper,
+ this.getConfiguration(), this.serverInfo.getServerName(),
+ logCloseThreadPool, masterRef);
+ this.splitLogWorkers.add(splitLogWorker);
+ splitLogWorker.start();
+ workersNeeded--;
+ } while (workersNeeded > 0);
+ }
+ }
+
+ /**
+ * Get the number of {@link SplitLogWorker} threads currently available.
+ *
+ * @return The number of {@link SplitLotWorker} threads currently running.
+ */
+ public int getNumberOfSplitLogWorkerThreads() {
+ return this.splitLogWorkers.size();
+ }
+
/*
* Verify that server is healthy
*/
@@ -4181,6 +4229,8 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
HConstants.HDFS_QUORUM_READ_THREADS_MAX, HConstants.DEFAULT_HDFS_QUORUM_READ_THREADS_MAX);
long timeout = conf.getLong(
HConstants.HDFS_QUORUM_READ_TIMEOUT_MILLIS, HConstants.DEFAULT_HDFS_QUORUM_READ_TIMEOUT_MILLIS);
+ int numSplitLogWorkers = conf.getInt(
+ HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, HConstants.DEFAULT_HREGIONSERVER_SPLITLOG_WORKERS_NUM);
boolean origProfiling = enableServerSideProfilingForAllCalls.get();
boolean newProfiling = conf.getBoolean(
@@ -4201,6 +4251,11 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
this.setHDFSQuorumReadTimeoutMillis(timeout);
}
+ if (numSplitLogWorkers != this.splitLogWorkers.size()) {
+ LOG.info("Number of SplitLogWorker threads is changed from " + this.splitLogWorkers.size() + " to " + numSplitLogWorkers);
+ this.setNumberOfSplitLogWorkerThreads(numSplitLogWorkers);
+ }
+
HRegionServer.useSeekNextUsingHint =
conf.getBoolean("hbase.regionserver.scan.timestampfilter.allow_seek_next_using_hint", true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfa65f89/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
index 3b3ac28..2f2b733 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
@@ -364,5 +364,24 @@ public class TestRegionServerOnlineConfigChange extends TestCase {
assertEquals(tableDescPropCustomVal, s1.conf.get(tableDescProp));
assertEquals(tableDescPropCustomVal, s2.conf.get(tableDescProp));
}
-}
+ public void testSetSplitLogWorkerThreadCount() {
+ int firstLogWorkerLimit = 5;
+ int secondLogWorkerLimit = 10;
+ int thirdLogWorkerLimit = 3;
+
+ assertEquals(HConstants.DEFAULT_HREGIONSERVER_SPLITLOG_WORKERS_NUM, rs1.getNumberOfSplitLogWorkerThreads());
+
+ conf.setInt(HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, firstLogWorkerLimit);
+ HRegionServer.configurationManager.notifyAllObservers(conf);
+ assertEquals(firstLogWorkerLimit, rs1.getNumberOfSplitLogWorkerThreads());
+
+ conf.setInt(HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, secondLogWorkerLimit);
+ HRegionServer.configurationManager.notifyAllObservers(conf);
+ assertEquals(secondLogWorkerLimit, rs1.getNumberOfSplitLogWorkerThreads());
+
+ conf.setInt(HConstants.HREGIONSERVER_SPLITLOG_WORKERS_NUM, thirdLogWorkerLimit);
+ HRegionServer.configurationManager.notifyAllObservers(conf);
+ assertEquals(thirdLogWorkerLimit, rs1.getNumberOfSplitLogWorkerThreads());
+ }
+}
[27/50] [abbrv] git commit: [master] Show hostname:port in
AssignmentPlan for bad machines
Posted by ec...@apache.org.
[master] Show hostname:port in AssignmentPlan for bad machines
Summary: Call `getHostNameWithPort` to change IP to hostname.
Test Plan: Local web server.
Reviewers: elliott, gauravm, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1399266
Tasks: 4571191
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43098 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0eccb58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0eccb58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0eccb58
Branch: refs/heads/0.89-fb
Commit: f0eccb58d4bb7415beedd3add079169a6452e6ea
Parents: 2e5a0d8
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 01:08:36 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
src/main/resources/hbase-webapps/master/assignmentPlan.jsp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0eccb58/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
index cb6ddbc..9a4d7c3 100644
--- a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
+++ b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
@@ -132,7 +132,7 @@
AssignmentPlan.POSITION favoredNodePosition =
AssignmentPlan.getFavoredServerPosition(favoredNodeList, currentRS);
if (favoredNodePosition == null) {
- position = "" + currentRS;
+ position = currentRS.getHostNameWithPort();
} else {
position = favoredNodePosition.toString();
if (favoredNodePosition == AssignmentPlan.POSITION.PRIMARY) {
[38/50] [abbrv] git commit: [master] Refactor copyTable a lil' bit
and pass taking config parameters from CLI.
Posted by ec...@apache.org.
[master] Refactor copyTable a lil' bit and pass taking config parameters from CLI.
Summary: this is needed for testing migration job for ods prod data, please take a look :)
Test Plan: try it out on shadow
Reviewers: rshroff, fan, gauravm, daviddeng, arjen, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@, paultuckfield
Differential Revision: https://phabricator.fb.com/D1430360
Tasks: 4682090
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43444 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2cb3ebbc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2cb3ebbc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2cb3ebbc
Branch: refs/heads/0.89-fb
Commit: 2cb3ebbc2b8b84674d5a27bd2422a202cff524b5
Parents: fede35d
Author: adela <ad...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jul 11 23:25:56 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/CopyTable.java | 267 ++++++++++---------
1 file changed, 134 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2cb3ebbc/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index d060d1e..1f6afa4 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -19,15 +19,20 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import java.io.IOException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
-import java.io.IOException;
-
/**
* Tool used to copy a table to another one which can be on a different setup.
* It is also configurable with a start and time as well as a specification
@@ -36,156 +41,152 @@ import java.io.IOException;
public class CopyTable {
final static String NAME = "Copy Table";
- static String rsClass = null;
- static String rsImpl = null;
- static long startTime = 0;
- static long endTime = 0;
- static String tableName = null;
- static String newTableName = null;
- static String peerAddress = null;
- static String families = null;
+ private final String rsClass;
+ private final String rsImpl;
+ private final long startTime;
+ private final long endTime;
+ private final String tableName;
+ private final String newTableName;
+ private final String peerAddress;
+ private final String[] families;
+
+
+ public CopyTable(String rsClass, String rsImpl, long startTime, long endTime,
+ String tableName, String newTableName, String zookeeperQuorum,
+ String[] families) {
+ this.rsClass = rsClass;
+ this.rsImpl = rsImpl;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.tableName = tableName;
+ this.newTableName = newTableName;
+ this.peerAddress = zookeeperQuorum;
+ this.families = families;
+ }
/**
* Sets up the actual job.
*
- * @param conf The current configuration.
- * @param args The command line parameters.
+ * @param conf
+ * The current configuration.
+ * @param args
+ * The command line parameters.
* @return The newly created job.
- * @throws IOException When setting up the job fails.
+ * @throws IOException
+ * When setting up the job fails.
*/
public static Job createSubmittableJob(Configuration conf, String[] args)
- throws IOException {
- if (!doCommandLine(args)) {
+ throws IOException {
+ Job job = null;
+ CopyTable cpTbl = null;
+ try {
+ cpTbl = buildCopyTableFromArguments(conf, args);
+ } catch (ParseException e) {
+ e.printStackTrace();
return null;
}
- Job job = new Job(conf, NAME + "_" + tableName);
- job.setJarByClass(CopyTable.class);
- Scan scan = new Scan();
- if (startTime != 0) {
- scan.setTimeRange(startTime,
- endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
- }
- if(families != null) {
- String[] fams = families.split(",");
- for(String fam : fams) {
- scan.addFamily(Bytes.toBytes(fam));
+ if (cpTbl != null) {
+ job = new Job(conf, NAME + "_" + cpTbl.tableName);
+ job.setJarByClass(CopyTable.class);
+ Scan scan = new Scan();
+ if (cpTbl.startTime != 0) {
+ scan.setTimeRange(cpTbl.startTime,
+ cpTbl.endTime == 0 ? HConstants.LATEST_TIMESTAMP : cpTbl.endTime);
+ }
+ if (cpTbl.families != null) {
+ for (String fam : cpTbl.families) {
+ scan.addFamily(Bytes.toBytes(fam));
+ }
}
+ TableMapReduceUtil.initTableMapperJob(cpTbl.tableName, scan,
+ Import.Importer.class, null, null, job);
+ TableMapReduceUtil.initTableReducerJob(
+ cpTbl.newTableName == null ? cpTbl.tableName : cpTbl.newTableName,
+ null, job, null, cpTbl.peerAddress, cpTbl.rsClass, cpTbl.rsImpl);
+ job.setNumReduceTasks(0);
+ } else {
+ System.out.println("CopyTable object is null, exiting!");
+ return null;
}
- TableMapReduceUtil.initTableMapperJob(tableName, scan,
- Import.Importer.class, null, null, job);
- TableMapReduceUtil.initTableReducerJob(
- newTableName == null ? tableName : newTableName, null, job,
- null, peerAddress, rsClass, rsImpl);
- job.setNumReduceTasks(0);
return job;
}
- /*
- * @param errorMsg Error message. Can be null.
- */
- private static void printUsage(final String errorMsg) {
- if (errorMsg != null && errorMsg.length() > 0) {
- System.err.println("ERROR: " + errorMsg);
- }
- System.err.println("Usage: CopyTable [--rs.class=CLASS] " +
- "[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] " +
- "[--new.name=NEW] [--peer.adr=ADR] <tablename>");
- System.err.println();
- System.err.println("Options:");
- System.err.println(" rs.class hbase.regionserver.class of the peer cluster");
- System.err.println(" specify if different from current cluster");
- System.err.println(" rs.impl hbase.regionserver.impl of the peer cluster");
- System.err.println(" starttime beginning of the time range");
- System.err.println(" without endtime means from starttime to forever");
- System.err.println(" endtime end of the time range");
- System.err.println(" new.name new table's name");
- System.err.println(" peer.adr Address of the peer cluster given in the format");
- System.err.println(" hbase.zookeeer.quorum:zookeeper.znode.parent");
- System.err.println(" families comma-seperated list of families to copy");
- System.err.println();
- System.err.println("Args:");
- System.err.println(" tablename Name of the table to copy");
- System.err.println();
- System.err.println("Examples:");
- System.err.println(" To copy 'TestTable' to a cluster that uses replication for a 1 hour window:");
- System.err.println(" $ bin/hbase " +
- "org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface " +
- "--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer --starttime=1265875194289 --endtime=1265878794289 " +
- "--peer.adr=server1,server2,server3:/hbase TestTable ");
+ private static void printHelp(Options opt) {
+ String example = "To copy 'TestTable' to a cluster that uses replication for a 1 hour window:\n $ bin/hbase "
+ + "org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface "
+ + "--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer --starttime=1265875194289 --endtime=1265878794289 "
+ + "--peer.adr=server1,server2,server3:/hbase TestTable ";
+ new HelpFormatter().printHelp(
+ "Job needs to be run on the source cluster \n CopyTable < tablename | -h > [--rs.class=CLASS] "
+ + "[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] "
+ + "[--new.name=NEW] [--peer.adr=ADR] [--D]", "", opt,
+ "Example: " + example);
}
- private static boolean doCommandLine(final String[] args) {
- // Process command-line args. TODO: Better cmd-line processing
- // (but hopefully something not as painful as cli options).
- if (args.length < 1) {
- printUsage(null);
- return false;
- }
- try {
- for (int i = 0; i < args.length; i++) {
- String cmd = args[i];
- if (cmd.equals("-h") || cmd.startsWith("--h")) {
- printUsage(null);
- return false;
- }
-
- final String rsClassArgKey = "--rs.class=";
- if (cmd.startsWith(rsClassArgKey)) {
- rsClass = cmd.substring(rsClassArgKey.length());
- continue;
- }
-
- final String rsImplArgKey = "--rs.impl=";
- if (cmd.startsWith(rsImplArgKey)) {
- rsImpl = cmd.substring(rsImplArgKey.length());
- continue;
- }
-
- final String startTimeArgKey = "--starttime=";
- if (cmd.startsWith(startTimeArgKey)) {
- startTime = Long.parseLong(cmd.substring(startTimeArgKey.length()));
- continue;
- }
-
- final String endTimeArgKey = "--endtime=";
- if (cmd.startsWith(endTimeArgKey)) {
- endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
- continue;
- }
-
- final String newNameArgKey = "--new.name=";
- if (cmd.startsWith(newNameArgKey)) {
- newTableName = cmd.substring(newNameArgKey.length());
- continue;
- }
-
- final String peerAdrArgKey = "--peer.adr=";
- if (cmd.startsWith(peerAdrArgKey)) {
- peerAddress = cmd.substring(peerAdrArgKey.length());
- continue;
- }
-
- final String familiesArgKey = "--families=";
- if (cmd.startsWith(familiesArgKey)) {
- families = cmd.substring(familiesArgKey.length());
- continue;
- }
+ private static Options buildOptions() {
+ Options opt = new Options();
+ opt.addOption("rs.class", true,
+ "hbase.regionserver.class of the destination cluster");
+ opt.addOption("rs.impl", true,
+ "hbase.regionserver.impl of the destination cluster");
+ opt.addOption(
+ "starttime",
+ true,
+ "beginning of the time range, don't specify if you want to copy [-infinity, end)");
+ opt.addOption(
+ "endtime",
+ true,
+ "end of the time range, don't specify if you want to copy the data [start, infinity)");
+ opt.addOption("new.name", true,
+ "name of the new table on the destination cluster");
+ opt.addOption("peer.adr", true,
+ "zookeeper.quorum of the destination cluster");
+ opt.addOption("families", true, "comma-seperated list of families to copy");
+ opt.addOption("tablename", true,
+ "name of the table that we are copying from the source cluster");
+ opt.addOption("D", true, "configuration override, provide arguments in format conf1=x, conf2=y, etc.");
+ opt.addOption("-h", false, "help");
+ return opt;
+ }
- if (i == args.length-1) {
- tableName = cmd;
+ /**
+ * Construct CopyTable object from the arguments from command line
+ */
+ private static CopyTable buildCopyTableFromArguments(Configuration conf,
+ final String[] args) throws ParseException {
+ Options opt = buildOptions();
+ CommandLine cmd = new GnuParser().parse(opt, args);
+ String rsClass = cmd.getOptionValue("rs.class");
+ String rsImpl = cmd.getOptionValue("rs.impl");
+ String startTimeString = cmd.getOptionValue("starttime");
+ long startTime = startTimeString == null ? 0 : Integer
+ .valueOf(startTimeString);
+ String endTimeString = cmd.getOptionValue("endtime");
+ long endTime = endTimeString == null ? 0 : Integer
+ .valueOf(endTimeString);
+ String newTableName = cmd.getOptionValue("new.name");
+ String zookeeperQuorum = cmd.getOptionValue("peer.adr");
+ String[] families = cmd.getOptionValues("families");
+ String tableName = cmd.getOptionValue("tablename");
+ if (tableName == null && zookeeperQuorum == null) {
+ printHelp(opt);
+ throw new ParseException(
+ "tableName OR zookeperQuorum (peer.adr) must be specified");
+ }
+ if (cmd.hasOption("D")) {
+ for (String confOpt : cmd.getOptionValues("D")) {
+ String[] kv = confOpt.split("=", 2);
+ if (kv.length == 2) {
+ conf.set(kv[0], kv[1]);
+ System.out.println("-D configuration override: " + kv[0] + "="
+ + kv[1]);
+ } else {
+ throw new ParseException("-D option format invalid: " + confOpt);
}
}
- if (newTableName == null && peerAddress == null) {
- printUsage("At least a new table name or a " +
- "peer address must be specified");
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- printUsage("Can't start because " + e.getMessage());
- return false;
}
- return true;
+ return new CopyTable(rsClass, rsImpl, startTime, endTime, tableName,
+ newTableName, zookeeperQuorum, families);
}
/**
[26/50] [abbrv] git commit: [HBASE-11383] Add jitter to HLog rolling
period
Posted by ec...@apache.org.
[HBASE-11383] Add jitter to HLog rolling period
Summary: Just in case things go wrong all at once make sure not to ddos the nn by making the rate limiting jittered.
Test Plan: mvn clean test -Dtest=TestHLog
Reviewers: adela, daviddeng
Reviewed By: daviddeng
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1393915
Tasks: 4549813
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43016 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/13c94a30
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/13c94a30
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/13c94a30
Branch: refs/heads/0.89-fb
Commit: 13c94a30b64ee58e60515129d4dcadae77e37124
Parents: d8ba450
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jun 20 17:28:45 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/regionserver/LogRoller.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c94a30/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 1eb0944..92f6cec 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.commons.lang.math.RandomUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
@@ -67,8 +68,10 @@ public class LogRoller extends HasThread implements LogRollListener {
public LogRoller(final HRegionServer server, int hlogIndexID) {
super();
this.server = server;
- this.rollperiod =
- this.server.conf.getLong("hbase.regionserver.logroll.period", 3600000);
+
+ long confRoll = this.server.conf.getLong("hbase.regionserver.logroll.period", 3600000);
+ this.rollperiod = (long) (confRoll + (RandomUtils.nextDouble() * 0.2 * confRoll));
+
this.timeBetweenRequest = this.server.conf.getLong(LOG_ROLL_REQUEST_PERIOD_KEY,
LOG_ROLL_REQUEST_PERIOD_DEFAULT);
this.hlogIndexID = hlogIndexID;
[21/50] [abbrv] git commit: [master] Show the stack track of the
exception
Posted by ec...@apache.org.
[master] Show the stack track of the exception
Summary: Instead of appending the error message to the message to be logged, put an Exception as the second argument.
Test Plan: No.
Reviewers: adela
Reviewed By: adela
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1386381
Tasks: 4525700
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42909 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/18a5653d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/18a5653d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/18a5653d
Branch: refs/heads/0.89-fb
Commit: 18a5653da755e45ab23398d43117176a06d0b4bd
Parents: bd01833
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 17 00:37:29 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/RegionPlacement.java | 30 +++++++++-----------
1 file changed, 14 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/18a5653d/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java b/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
index 6c7e7aa..217b822 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
@@ -131,7 +131,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
plan.initialize(primaryRSMap, secondaryAndTertiaryRSMap);
return plan;
} catch (Exception e) {
- LOG.debug("Cannot generate the assignment plan because " + e);
+ LOG.debug("Cannot generate the assignment plan", e);
return null;
}
}
@@ -401,8 +401,8 @@ public class RegionPlacement implements RegionPlacementPolicy{
+ regionInfo.getRegionNameAsString());
}
} catch (Exception e) {
- LOG.warn("Cannot place the favored nodes for region " +
- regionInfo.getRegionNameAsString() + " because " + e);
+ LOG.warn("Cannot place the favored nodes for region "
+ + regionInfo.getRegionNameAsString(), e);
continue;
}
}
@@ -460,12 +460,12 @@ public class RegionPlacement implements RegionPlacementPolicy{
serverSkipSet);
if (secondaryRS == null || tertiaryRS == null) {
- LOG.error("Cannot place the secondary and terinary"
+ LOG.error("Cannot place the secondary and terinary "
+ "region server for region "
+ regionInfo.getRegionNameAsString());
}
// Create the secondary and tertiary pair
- pair = new Pair<HServerAddress, HServerAddress>();
+ pair = new Pair<>();
pair.setFirst(secondaryRS);
pair.setSecond(tertiaryRS);
}
@@ -537,13 +537,13 @@ public class RegionPlacement implements RegionPlacementPolicy{
+ regionInfo.getRegionNameAsString());
}
// Create the secondary and tertiary pair
- pair = new Pair<HServerAddress, HServerAddress>();
+ pair = new Pair<>();
pair.setFirst(secondaryRS);
pair.setSecond(tertiaryRS);
} else {
// Pick the secondary rs from this secondary rack
// and pick the tertiary from another random rack
- pair = new Pair<HServerAddress, HServerAddress>();
+ pair = new Pair<>();
HServerAddress secondary = domain.getOneRandomServer(secondaryRack);
pair.setFirst(secondary);
@@ -572,7 +572,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
}
} catch (Exception e) {
LOG.warn("Cannot place the favored nodes for region "
- + regionInfo.getRegionNameAsString() + " because " + e);
+ + regionInfo.getRegionNameAsString(), e);
continue;
}
}
@@ -1163,8 +1163,8 @@ public class RegionPlacement implements RegionPlacementPolicy{
genAssignmentPlan(table, assignmentSnapshot, regionLocalityMap, plan,
USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY);
} catch (Exception e) {
- LOG.error("Get some exceptions for placing primary region server" +
- "for table " + table + " because " + e);
+ LOG.error("Get some exceptions for placing primary region server"
+ + "for table " + table, e);
}
}
LOG.info("Finish to generate the new assignment plan for the " +
@@ -1217,8 +1217,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
metaTable.put(puts);
LOG.info("Updated the META with the new assignment plan");
} catch (Exception e) {
- LOG.error("Failed to update META with the new assignment" +
- "plan because " + e.getMessage());
+ LOG.error("Failed to update META with the new assignment plan", e);
}
}
@@ -1893,7 +1892,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
try {
favoredNodes = RegionPlacement.getFavoredNodeList(favoredNodesStr);
} catch (IllegalArgumentException e) {
- LOG.error("Cannot parse the invalid favored nodes because " + e);
+ LOG.error("Cannot parse the invalid favored nodes", e);
}
AssignmentPlan newPlan = new AssignmentPlan();
newPlan.updateAssignmentPlan(regionInfo, favoredNodes);
@@ -1944,8 +1943,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
ObjectMapper jsonMapper = new ObjectMapper();
jsonMapper.defaultPrettyPrintingWriter().writeValue(new File(path), data);
} catch (Exception e) {
- LOG.error("Unable to download current assignment plan" + e);
- e.printStackTrace();
+ LOG.error("Unable to download current assignment plan", e);
}
} else {
printHelp(opt);
@@ -2358,7 +2356,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
try {
favoredNodes = RegionPlacement.getFavoredNodeList(nodesStr);
} catch (IllegalArgumentException e) {
- LOG.error("Cannot parse the invalid favored nodes because " + e);
+ LOG.error("Cannot parse the invalid favored nodes", e);
}
}
newPlan.updateAssignmentPlan(regionInfo, favoredNodes);
[04/50] [abbrv] git commit: [HBASE-10698] Updated the Thrift IDL file
to inclue the Async/Sync and Put annotation changes
Posted by ec...@apache.org.
[HBASE-10698] Updated the Thrift IDL file to inclue the Async/Sync and Put annotation changes
Summary: Updating the IDL file and the IDL generating script.
Test Plan: N/A
Reviewers: elliott, manukranthk, daviddeng
Reviewed By: daviddeng
Subscribers: andrewcox, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1358886
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42609 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/290cd1cf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/290cd1cf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/290cd1cf
Branch: refs/heads/0.89-fb
Commit: 290cd1cf5f017ca1702584e56b1abfb0ab88098b
Parents: d5640c1
Author: gauravm <ga...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 2 17:31:52 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:43:00 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/thrift/HBase.thrift | 43 ++++++++------------
.../hbase/thrift/gen_thrift_from_swift.sh | 2 +-
2 files changed, 19 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/290cd1cf/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
index 42171fc..9d5ee77 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
@@ -17,10 +17,9 @@ struct KeyValue {
struct Put {
1: binary row;
- 2: i64 ts;
- 3: map<binary, list<KeyValue>> familyMap;
- 4: i64 lockId;
- 5: bool writeToWAL;
+ 2: map<binary, list<KeyValue>> familyMap;
+ 3: i64 lockId;
+ 4: bool writeToWAL;
}
struct TimeRange {
@@ -211,39 +210,34 @@ service ThriftHRegionInterface {
bool checkAndPut(1: binary regionName, 2: binary row, 3: binary family, 4: binary qualifier, 5: binary value, 6: Put put) throws (1: ThriftHBaseException ex1);
void close(1: i64 scannerId) throws (1: ThriftHBaseException ex1);
void closeRegion(1: HRegionInfo hri, 2: bool reportWhenCompleted) throws (1: ThriftHBaseException ex1);
- void deleteAsync(1: binary regionName, 2: Delete deleteArg) throws (1: ThriftHBaseException ex1);
bool exists(1: binary regionName, 2: Get get) throws (1: ThriftHBaseException ex1);
void flushRegion(1: binary regionName) throws (1: ThriftHBaseException ex1);
void flushRegionIfOlderThanTS(1: binary regionName, 2: i64 ifOlderThanTS) throws (1: ThriftHBaseException ex1);
- Result getAsync(1: binary regionName, 2: Get get) throws (1: ThriftHBaseException ex1);
Result getClosestRowBefore(1: binary regionName, 2: binary row, 3: binary family) throws (1: ThriftHBaseException ex1);
- Result getClosestRowBeforeAsync(1: binary regionName, 2: binary row, 3: binary family) throws (1: ThriftHBaseException ex1);
- string getConfProperty(1: string arg0) throws (1: ThriftHBaseException ex1);
- i64 getCurrentTimeMillis();
+ string getConfProperty(1: string paramName) throws (1: ThriftHBaseException ex1);
+ i64 getCurrentTimeMillis() throws (1: ThriftHBaseException ex1);
list<string> getHLogsList(1: bool rollCurrentHLog) throws (1: ThriftHBaseException ex1);
HServerInfo getHServerInfo() throws (1: ThriftHBaseException ex1);
- list<Bucket> getHistogram(1: binary arg0) throws (1: ThriftHBaseException ex1);
- list<Bucket> getHistogramForStore(1: binary arg0, 2: binary arg1) throws (1: ThriftHBaseException ex1);
- list<list<Bucket>> getHistograms(1: list<binary> arg0) throws (1: ThriftHBaseException ex1);
- i64 getLastFlushTime(1: binary regionName);
- map<binary, i64> getLastFlushTimes();
+ list<Bucket> getHistogram(1: binary regionName) throws (1: ThriftHBaseException ex1);
+ list<Bucket> getHistogramForStore(1: binary regionName, 2: binary family) throws (1: ThriftHBaseException ex1);
+ list<list<Bucket>> getHistograms(1: list<binary> regionNames) throws (1: ThriftHBaseException ex1);
+ i64 getLastFlushTime(1: binary regionName) throws (1: ThriftHBaseException ex1);
+ map<binary, i64> getLastFlushTimes() throws (1: ThriftHBaseException ex1);
HRegionLocation getLocation(1: binary tableName, 2: binary row, 3: bool reload) throws (1: ThriftHBaseException ex1);
HRegionInfo getRegionInfo(1: binary regionName) throws (1: ThriftHBaseException ex1);
list<HRegionInfo> getRegionsAssignment() throws (1: ThriftHBaseException ex1);
list<Result> getRows(1: binary regionName, 2: list<Get> gets) throws (1: ThriftHBaseException ex1);
- i64 getStartCode();
- string getStopReason();
+ i64 getStartCode() throws (1: ThriftHBaseException ex1);
+ string getStopReason() throws (1: ThriftHBaseException ex1);
list<string> getStoreFileList(1: binary regionName, 2: binary columnFamily) throws (1: ThriftHBaseException ex1);
list<string> getStoreFileListForAllColumnFamilies(1: binary regionName) throws (1: ThriftHBaseException ex1);
list<string> getStoreFileListForColumnFamilies(1: binary regionName, 2: list<binary> columnFamilies) throws (1: ThriftHBaseException ex1);
i64 incrementColumnValue(1: binary regionName, 2: binary row, 3: binary family, 4: binary qualifier, 5: i64 amount, 6: bool writeToWAL) throws (1: ThriftHBaseException ex1);
- bool isStopped();
- i64 lockRow(1: binary regionName, 2: binary row) throws (1: ThriftHBaseException ex1);
- RowLock lockRowAsync(1: binary regionName, 2: binary row) throws (1: ThriftHBaseException ex1);
+ bool isStopped() throws (1: ThriftHBaseException ex1);
+ RowLock lockRow(1: binary regionName, 2: binary row) throws (1: ThriftHBaseException ex1);
TMultiResponse multiAction(1: MultiAction multi) throws (1: ThriftHBaseException ex1);
MultiPutResponse multiPut(1: MultiPut puts) throws (1: ThriftHBaseException ex1);
void mutateRow(1: binary regionName, 2: TRowMutations arm) throws (1: ThriftHBaseException ex1);
- void mutateRowAsync(1: binary regionName, 2: TRowMutations arm) throws (1: ThriftHBaseException ex1);
void mutateRows(1: binary regionName, 2: list<TRowMutations> armList) throws (1: ThriftHBaseException ex1);
Result next(1: i64 scannerId) throws (1: ThriftHBaseException ex1);
list<Result> nextRows(1: i64 scannerId, 2: i32 numberOfRows) throws (1: ThriftHBaseException ex1);
@@ -256,12 +250,11 @@ service ThriftHRegionInterface {
bool scanClose(1: i64 id) throws (1: ThriftHBaseException ex1);
ScannerResult scanNext(1: i64 id, 2: i32 numberOfRows) throws (1: ThriftHBaseException ex1);
ScannerResult scanOpen(1: binary regionName, 2: Scan scan, 3: i32 numberOfRows) throws (1: ThriftHBaseException ex1);
- void setHDFSQuorumReadTimeoutMillis(1: i64 timeoutMillis);
- void setNumHDFSQuorumReadThreads(1: i32 maxThreads);
- void stop(1: string why);
- void stopForRestart();
+ void setHDFSQuorumReadTimeoutMillis(1: i64 timeoutMillis) throws (1: ThriftHBaseException ex1);
+ void setNumHDFSQuorumReadThreads(1: i32 maxThreads) throws (1: ThriftHBaseException ex1);
+ void stop(1: string why) throws (1: ThriftHBaseException ex1);
+ void stopForRestart() throws (1: ThriftHBaseException ex1);
void unlockRow(1: binary regionName, 2: i64 lockId) throws (1: ThriftHBaseException ex1);
- void unlockRowAsync(1: binary regionName, 2: i64 lockId) throws (1: ThriftHBaseException ex1);
void updateConfiguration() throws (1: ThriftHBaseException ex1);
i32 updateFavoredNodes(1: AssignmentPlan plan) throws (1: ThriftHBaseException ex1);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/290cd1cf/src/main/resources/org/apache/hadoop/hbase/thrift/gen_thrift_from_swift.sh
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/gen_thrift_from_swift.sh b/src/main/resources/org/apache/hadoop/hbase/thrift/gen_thrift_from_swift.sh
index d300f3a..89a8621 100755
--- a/src/main/resources/org/apache/hadoop/hbase/thrift/gen_thrift_from_swift.sh
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift/gen_thrift_from_swift.sh
@@ -11,4 +11,4 @@ export CLASSPATH=$CLASSPATH:$1:$2:$3
dirpath=`dirname $0`
-$JAVA_HOME/bin/java com.facebook.swift.generator.swift2thrift.Main -allow_multiple_packages org.apache.hadoop.hbase org.apache.hadoop.hbase.KeyValue org.apache.hadoop.hbase.client.Put org.apache.hadoop.hbase.io.TimeRange org.apache.hadoop.hbase.filter.TFilter org.apache.hadoop.hbase.client.Get org.apache.hadoop.hbase.client.MultiPut org.apache.hadoop.hbase.client.Delete org.apache.hadoop.hbase.client.Scan org.apache.hadoop.hbase.HColumnDescriptor org.apache.hadoop.hbase.HTableDescriptor org.apache.hadoop.hbase.HRegionInfo org.apache.hadoop.hbase.client.MultiPutResponse org.apache.hadoop.hbase.client.Result org.apache.hadoop.hbase.HServerAddress 'org.apache.hadoop.hbase.HServerLoad$RegionLoad' org.apache.hadoop.hbase.HServerLoad org.apache.hadoop.hbase.HServerInfo org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException org.apache.hadoop.hbase.client.MultiAction 'org.apache.hadoop.hbase.client.IntegerOrResultOrException$Type' org.apache.hadoop.hbase.client.IntegerOrResultOr
Exception org.apache.hadoop.hbase.client.TMultiResponse org.apache.hadoop.hbase.client.TRowMutations org.apache.hadoop.hbase.master.AssignmentPlan org.apache.hadoop.hbase.client.RowLock 'org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram$HFileStat' 'org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram$Bucket' org.apache.hadoop.hbase.HRegionLocation org.apache.hadoop.hbase.ipc.ScannerResult org.apache.hadoop.hbase.ipc.ThriftHRegionInterface -out "$dirpath/HBase.thrift" -namespace cpp facebook.hbase.hbcpp -namespace php hbase
+$JAVA_HOME/bin/java com.facebook.swift.generator.swift2thrift.Main -allow_multiple_packages org.apache.hadoop.hbase org.apache.hadoop.hbase.KeyValue org.apache.hadoop.hbase.client.Put org.apache.hadoop.hbase.io.TimeRange org.apache.hadoop.hbase.filter.TFilter org.apache.hadoop.hbase.client.Get org.apache.hadoop.hbase.client.MultiPut org.apache.hadoop.hbase.client.Delete org.apache.hadoop.hbase.client.Scan org.apache.hadoop.hbase.HColumnDescriptor org.apache.hadoop.hbase.HTableDescriptor org.apache.hadoop.hbase.HRegionInfo org.apache.hadoop.hbase.client.MultiPutResponse org.apache.hadoop.hbase.client.Result org.apache.hadoop.hbase.HServerAddress 'org.apache.hadoop.hbase.HServerLoad$RegionLoad' org.apache.hadoop.hbase.HServerLoad org.apache.hadoop.hbase.HServerInfo org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException org.apache.hadoop.hbase.client.MultiAction 'org.apache.hadoop.hbase.client.IntegerOrResultOrException$Type' org.apache.hadoop.hbase.client.IntegerOrResultOr
Exception org.apache.hadoop.hbase.client.TMultiResponse org.apache.hadoop.hbase.client.TRowMutations org.apache.hadoop.hbase.master.AssignmentPlan org.apache.hadoop.hbase.client.RowLock 'org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram$HFileStat' 'org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram$Bucket' org.apache.hadoop.hbase.HRegionLocation org.apache.hadoop.hbase.ipc.ScannerResult 'org.apache.hadoop.hbase.ipc.ThriftHRegionInterface$Sync' -out "$dirpath/HBase.thrift" -namespace cpp facebook.hbase.hbcpp -namespace php hbase
[09/50] [abbrv] git commit: [master] Clear rootRegionLocation if the
root is not served there
Posted by ec...@apache.org.
[master] Clear rootRegionLocation if the root is not served there
Summary: Clear `TableServers.rootRegionLocation` when receiving `NotServingRegionException`
Test Plan: `TestGetRegionLocation`
Reviewers: elliott, fan, manukranthk
Reviewed By: fan, manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1367998
Tasks: 4448200
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42760 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df1f91fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df1f91fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df1f91fa
Branch: refs/heads/0.89-fb
Commit: df1f91fad67a85315a80ff39beca640ea33ff4f4
Parents: 91eea3f
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 9 19:02:34 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 2 +-
.../apache/hadoop/hbase/HTableDescriptor.java | 4 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 4 +-
.../hadoop/hbase/client/HTableMultiplexer.java | 24 +++----
.../hadoop/hbase/client/TableServers.java | 20 +++---
.../hadoop/hbase/HBaseTestingUtility.java | 31 +++++++++
.../regionserver/TestGetRegionLocation.java | 69 +++++++++++++++++++-
7 files changed, 126 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index dee47e0..9b30994 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -518,7 +518,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return this.tableDesc.isRootRegion();
}
- /** @return true if this is the meta table */
+ /** @return true if this is the META table */
public boolean isMetaTable() {
return this.tableDesc.isMetaTable();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 6f9b81a..5ddddfb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -35,7 +35,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
-import com.google.common.collect.ImmutableSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +46,7 @@ import org.apache.hadoop.io.WritableComparable;
import com.facebook.swift.codec.ThriftConstructor;
import com.facebook.swift.codec.ThriftField;
import com.facebook.swift.codec.ThriftStruct;
+import com.google.common.collect.ImmutableSet;
/**
* HTableDescriptor contains the name of an HTable, and its
@@ -324,7 +324,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
values.put(IS_META_KEY, isMeta? TRUE: FALSE);
}
- /** @return true if table is the meta table */
+ /** @return true if table is the META table */
public boolean isMetaTable() {
return isMetaRegion() && !isRootRegion();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ffeaf81..c16889b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -91,7 +91,7 @@ public class HBaseAdmin {
// use 30s pause time between retries instead of the small default
// pause time. Shouldn't the RPC read timeout be also increased?
this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, 30 * 1000);
- this.numRetries = conf.getInt("hbase.client.retries.number", 5);
+ this.numRetries = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
this.master = connection.getMaster();
}
@@ -1441,7 +1441,7 @@ public class HBaseAdmin {
public static void checkHBaseAvailable(Configuration conf)
throws MasterNotRunningException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
- copyOfConf.setInt("hbase.client.retries.number", 1);
+ copyOfConf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 1);
new HBaseAdmin(copyOfConf);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
index a3360c0..dfee8c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
@@ -57,13 +57,13 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* Each put will be sharded into different buffer queues based on its destination region server.
* So each region server buffer queue will only have the puts which share the same destination.
* And each queue will have a flush worker thread to flush the puts request to the region server.
- * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that
+ * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that
* particular queue.
- *
+ *
* Also all the puts will be retried as a configuration number before dropping.
* And the HTableMultiplexer can report the number of buffered requests and the number of the
* failed (dropped) requests in total or on per region server basis.
- *
+ *
* This class is thread safe.
*/
public class HTableMultiplexer {
@@ -87,11 +87,11 @@ public class HTableMultiplexer {
private final long frequency;
//initial number of threads in the pool
public static final int INITIAL_NUM_THREADS = 10;
-
+
/**
- *
+ *
* @param conf The HBaseConfiguration
- * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops
+ * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops
* for each region server before dropping the request.
*/
public HTableMultiplexer(Configuration conf, int perRegionServerBufferQueueSize) {
@@ -102,7 +102,7 @@ public class HTableMultiplexer {
this.serverToFlushWorkerMap = new ConcurrentHashMap<HServerAddress, HTableFlushWorker>();
this.tableNameToHTableMap = new ConcurrentSkipListMap<byte[], HTable>(
Bytes.BYTES_COMPARATOR);
- this.retryNum = conf.getInt("hbase.client.retries.number", 10);
+ this.retryNum = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 10);
this.retriedInQueueMax = conf.getInt("hbase.client.retried.inQueue", 10000);
this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize;
this.frequency = conf.getLong("hbase.htablemultiplexer.flush.frequency.ms",
@@ -127,7 +127,7 @@ public class HTableMultiplexer {
}
/**
- * The puts request will be buffered by their corresponding buffer queue.
+ * The puts request will be buffered by their corresponding buffer queue.
* Return the list of puts which could not be queued.
* @param table
* @param put
@@ -138,13 +138,13 @@ public class HTableMultiplexer {
HBaseRPCOptions options) throws IOException {
if (puts == null)
return null;
-
+
List <Put> failedPuts = null;
boolean result;
for (Put put : puts) {
result = put(table, put, this.retryNum, options);
if (result == false) {
-
+
// Create the failed puts list if necessary
if (failedPuts == null) {
failedPuts = new ArrayList<Put>();
@@ -183,7 +183,7 @@ public class HTableMultiplexer {
LinkedBlockingQueue<PutStatus> queue = getBufferedQueue(addr);
// Generate a MultiPutStatus obj and offer it into the queue
PutStatus s = new PutStatus(loc.getRegionInfo(), put, retry, options);
-
+
return queue.offer(s);
}
} catch (Exception e) {
@@ -480,7 +480,7 @@ public class HTableMultiplexer {
return this.tableNameToHTableMap.size();
}
}
-
+
private static class PutStatus {
private final HRegionInfo regionInfo;
private final Put put;
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index bc38cd6..bb671fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -51,11 +51,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.ListeningExecutorService;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -97,8 +94,11 @@ import org.apache.hadoop.ipc.RemoteException;
import org.apache.thrift.transport.TTransportException;
import com.google.common.base.Preconditions;
-
-import javax.annotation.Nullable;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningExecutorService;
/* Encapsulates finding the servers for an HBase instance */
public class TableServers implements ServerConnection {
@@ -1079,8 +1079,6 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
parentTable, row);
metaCache.add(tableName, location);
}
- } catch (Throwable t) {
- throw t;
} finally {
HBaseThriftRPC.isMeta.get().pop();
}
@@ -1094,6 +1092,10 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
// already processed this. Don't process this again.
throw e;
} catch (IOException e) {
+ if (e instanceof NotServingRegionException) {
+ this.rootRegionLocation = null;
+ }
+
if (e instanceof RemoteException) {
e = RemoteExceptionHandler
.decodeRemoteException((RemoteException) e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 101e552..10762fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1561,6 +1561,37 @@ public class HBaseTestingUtility {
return list;
}
+ /**
+ * @return the only HRegion for ROOT table. Returns a null if not found.
+ * It loops over all regions to find it, could be slow.
+ */
+ public HRegion getRootRegion() {
+ for (HRegionServer rs : getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ if (region.getRegionInfo().isRootRegion()) {
+ return region;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * @return one of HRegion for META table. Returns a null if not found.
+ * It loops over all regions to find it, could be slow.
+ */
+ public HRegion getMetaRegion() {
+ for (HRegionServer rs : getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ if (region.getRegionInfo().isMetaTable()) {
+ return region;
+ }
+ }
+ }
+
+ return null;
+ }
/**
* Wait until <code>countOfRegion</code> in .META. have a non-empty
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
index efb24b4..de391c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
@@ -30,13 +30,17 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.TableServers;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -48,7 +52,7 @@ public class TestGetRegionLocation {
private static final Log LOG = LogFactory.getLog(TestGetRegionLocation.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static Configuration CONF = TEST_UTIL.getConfiguration();
- private final static byte[] TABLE = Bytes.toBytes("table");
+ private final static StringBytes TABLE = new StringBytes("table");
private final static byte[] FAMILY = Bytes.toBytes("family");
private final static byte[][] FAMILIES = { FAMILY };
private final static byte[] START_KEY = Bytes.toBytes("aaa");
@@ -58,10 +62,13 @@ public class TestGetRegionLocation {
@Before
public void setUp() throws IOException, InterruptedException {
+ TEST_UTIL.getConfiguration().setInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
+
// Use assignment plan so that regions are not moved unexpectedly.
TEST_UTIL.useAssignmentLoadBalancer();
TEST_UTIL.startMiniCluster(NUM_SLAVES);
- TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS);
+ TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS)
+ .close();
}
@After
@@ -69,6 +76,64 @@ public class TestGetRegionLocation {
TEST_UTIL.shutdownMiniCluster();
}
+ @Test(timeout = 150000)
+ public void testMetaRootRegionRelocation() throws Exception {
+ Assert.assertTrue("We should have more than one region servers",
+ NUM_SLAVES >= 3);
+
+ try (TableServers connection = new TableServers(CONF)) {
+ HRegionLocation location =
+ connection.getRegionLocation(TABLE, HConstants.EMPTY_START_ROW, true);
+ Assert.assertNotNull("Cannot get location", location);
+
+ HRegion rootRegion = TEST_UTIL.getRootRegion();
+ Assert.assertNotNull("Cannot find ROOT region", rootRegion);
+ HRegionServer rootServer =
+ TEST_UTIL.getRSWithRegion(rootRegion.getRegionName());
+
+ HRegion metaRegion = TEST_UTIL.getMetaRegion();
+ Assert.assertNotNull("Cannot find META region", metaRegion);
+ HRegionServer metaServer =
+ TEST_UTIL.getRSWithRegion(metaRegion.getRegionName());
+
+ TEST_UTIL.getMiniHBaseCluster().getMaster()
+ .addServerToBlacklist(rootServer.getServerInfo().getHostnamePort());
+ TEST_UTIL.getMiniHBaseCluster().getMaster()
+ .addServerToBlacklist(metaServer.getServerInfo().getHostnamePort());
+
+ rootServer.closeRegion(rootRegion.getRegionInfo(), true);
+
+ metaServer.closeRegion(metaRegion.getRegionInfo(), true);
+
+ try {
+ rootServer.getRegion(rootRegion.getRegionName());
+ Assert.fail("Should throw NotServingRegionException");
+ } catch (NotServingRegionException e) {
+ // good
+ }
+ try {
+ metaServer.getRegion(metaRegion.getRegionName());
+ Assert.fail("Should throw NotServingRegionException");
+ } catch (NotServingRegionException e) {
+ // good
+ }
+
+ try {
+ connection.listTables();
+ } catch (RetriesExhaustedException e) {
+ e.printStackTrace();
+ Assert.fail("listTable failed: " + e);
+ }
+
+ try {
+ connection.listTables();
+ } catch (NotServingRegionException e) {
+ e.printStackTrace();
+ Assert.fail("listTable failed: " + e);
+ }
+ }
+ }
+
/**
* This test tests the getRegionLocation() call in the
* (Thrift)HRegionInterface API.
[39/50] [abbrv] git commit: [HBASE-11431] Add support of running from
command line for 'hbase shell'
Posted by ec...@apache.org.
[HBASE-11431] Add support of running from command line for 'hbase shell'
Summary: Process the arguments and run it.
Test Plan: Tested on dev cluster
Reviewers: gauravm, adela, elliott, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1409512
Tasks: 4571191
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43674 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/89949ea1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/89949ea1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/89949ea1
Branch: refs/heads/0.89-fb
Commit: 89949ea164f553949f18d7eac3fed792cf180dfe
Parents: 14758a6
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jul 24 17:07:05 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
bin/hirb.rb | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/89949ea1/bin/hirb.rb
----------------------------------------------------------------------
diff --git a/bin/hirb.rb b/bin/hirb.rb
index d6892b5..a1c6baa 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -54,9 +54,11 @@ cmdline_help = <<HERE # HERE document output as shell usage
HBase Shell command-line options:
format Formatter for outputting results: console | html. Default: console
-d | --debug Set DEBUG log levels.
+ --exec='cmd' Run the command and exit immediately.
HERE
found = []
format = 'console'
+cmdline = nil
script2run = nil
log_level = org.apache.log4j.Level::ERROR
for arg in ARGV
@@ -77,6 +79,11 @@ for arg in ARGV
log_level = org.apache.log4j.Level::DEBUG
$fullBackTrace = true
puts "Setting DEBUG log level..."
+ elsif arg =~ /^--exec=(.+)/i
+ cmdline = $1
+ puts "exec " + cmdline
+ found.push(arg)
+ break
else
# Presume it a script. Save it off for running later below
# after we've set up some environment.
@@ -146,6 +153,9 @@ end
# Include hbase constants
include HBaseConstants
+eval(cmdline) if cmdline
+exit if cmdline
+
# If script2run, try running it. Will go on to run the shell unless
# script calls 'exit' or 'exit 0' or 'exit errcode'.
load(script2run) if script2run
[17/50] [abbrv] git commit: [master] Fix
testNonCachedGetRegionLocation
Posted by ec...@apache.org.
[master] Fix testNonCachedGetRegionLocation
Summary: Move this case from `TestFromClientSide` to `TestGetRegionLocation`, `TEST_UTIL.useAssignmentLoadBalancer` is called which fixes the problem.
Test Plan: `TestGetRegionLocation`
Reviewers: manukranthk, fan, gauravm, rshroff, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1372191
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42813 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eae4c263
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eae4c263
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eae4c263
Branch: refs/heads/0.89-fb
Commit: eae4c2636aff929264ad2ffddbebed5417f761b2
Parents: ae31cc5
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 11 21:35:52 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/client/TestFromClientSide.java | 51 ----------------
.../regionserver/TestGetRegionLocation.java | 61 ++++++++++++++++++++
2 files changed, 61 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/eae4c263/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 2d4d0e3..c347154 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -4416,56 +4415,6 @@ public class TestFromClientSide {
compactCFTable(1);
}
- /**
- * Tests the non cached version of getRegionLocation by moving a region.
- */
- @Test(timeout = 180000)
- public void testNonCachedGetRegionLocation() throws Exception {
- // Test Initialization.
- String tableName = "testNonCachedGetRegionLocation";
- byte [] TABLE = Bytes.toBytes(tableName);
- byte [] family1 = Bytes.toBytes("f1");
- byte [] family2 = Bytes.toBytes("f2");
- HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
- HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
- Map <HRegionInfo, HServerAddress> regionsMap = table.getRegionsInfo();
- assertEquals(1, regionsMap.size());
- HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
- HServerAddress addrBefore = regionsMap.get(regionInfo);
- // Verify region location before move.
- HServerAddress addrCache =
- table.getRegionLocation(regionInfo.getStartKey()).getServerAddress();
- HServerAddress addrNoCache =
- table.getRegionLocation(regionInfo.getStartKey(),
- true).getServerAddress();
-
- assertEquals(addrBefore.getPort(), addrCache.getPort());
- assertEquals(addrBefore.getPort(), addrNoCache.getPort());
-
- HServerAddress addrAfter = null;
- // Now move the region to a different server.
- for (int i = 0; i < SLAVES; i++) {
- HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
- HServerAddress addr = regionServer.getServerInfo().getServerAddress();
- if (addr.getPort() != addrBefore.getPort()) {
- TEST_UTIL.moveRegionAndAssignment(regionInfo, addr);
- TEST_UTIL.waitForTableConsistent();
- addrAfter = addr;
- break;
- }
- }
-
- // Verify the region was moved.
- addrCache =
- table.getRegionLocation(regionInfo.getStartKey()).getServerAddress();
- addrNoCache =
- table.getRegionLocation(regionInfo.getStartKey(),
- true).getServerAddress();
- assertNotNull(addrAfter);
- assertTrue(addrAfter.getPort() != addrCache.getPort());
- assertEquals(addrAfter.getPort(), addrNoCache.getPort());
- }
-
@Test(timeout = 180000)
public void testAtomicRowMutation() throws Exception {
LOG.info("Starting testAtomicRowMutation");
http://git-wip-us.apache.org/repos/asf/hbase/blob/eae4c263/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
index de391c5..3d59361 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
@@ -20,6 +20,10 @@
package org.apache.hadoop.hbase.regionserver;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -33,9 +37,11 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.TableServers;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -260,4 +266,59 @@ public class TestGetRegionLocation {
}
}
+ /**
+ * Tests the non cached version of getRegionLocation by moving a region.
+ */
+ @Test(timeout = 180000)
+ public void testNonCachedGetRegionLocation() throws Exception {
+ // Test Initialization.
+ String tableName = "testNonCachedGetRegionLocation";
+ byte[] TABLE = Bytes.toBytes(tableName);
+ byte[] family1 = Bytes.toBytes("f1");
+ byte[] family2 = Bytes.toBytes("f2");
+ try (HTable table =
+ TEST_UTIL.createTable(TABLE, new byte[][] { family1, family2 }, 10)) {
+ Map<HRegionInfo, HServerAddress> regionsMap = table.getRegionsInfo();
+ assertEquals("Number of regions", 1, regionsMap.size());
+ HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
+ HServerAddress addrBefore = regionsMap.get(regionInfo);
+ // Verify region location before move.
+ HServerAddress addrCache =
+ table.getRegionLocation(regionInfo.getStartKey()).getServerAddress();
+ HServerAddress addrNoCache =
+ table.getRegionLocation(regionInfo.getStartKey(), true)
+ .getServerAddress();
+
+ assertEquals("Port(cached)", addrBefore.getPort(), addrCache.getPort());
+ assertEquals("Port(no-cached)", addrBefore.getPort(),
+ addrNoCache.getPort());
+
+ HServerAddress addrAfter = null;
+ // Now move the region to a different server.
+ for (int i = 0; i < NUM_SLAVES; i++) {
+ HRegionServer regionServer =
+ TEST_UTIL.getHBaseCluster().getRegionServer(i);
+ HServerAddress addr = regionServer.getServerInfo().getServerAddress();
+ if (addr.getPort() != addrBefore.getPort()) {
+ TEST_UTIL.moveRegionAndAssignment(regionInfo, addr);
+ TEST_UTIL.waitForTableConsistent();
+ addrAfter = addr;
+ break;
+ }
+ }
+
+ // Verify the region was moved.
+ addrCache =
+ table.getRegionLocation(regionInfo.getStartKey()).getServerAddress();
+ addrNoCache =
+ table.getRegionLocation(regionInfo.getStartKey(), true)
+ .getServerAddress();
+ assertNotNull("Not server for destination", addrAfter);
+ assertTrue("Server is the same as before",
+ addrAfter.getPort() != addrCache.getPort());
+ assertEquals("Port after region mored", addrAfter.getPort(),
+ addrNoCache.getPort());
+ }
+ }
+
}
[19/50] [abbrv] git commit: [HBASE-9930] Considering TimeoutException
as equivalent of SocketTimeoutException.
Posted by ec...@apache.org.
[HBASE-9930] Considering TimeoutException as equivalent of SocketTimeoutException.
Summary: While running bellatrix on the Titanshadow tier, I found that the TimeoutException was not being considered for PFFE. This diff is to incorporate TimeoutException too.
Test Plan: Minor Change.
Reviewers: aaiyer, rshroff, adela, fan
Reviewed By: fan
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1392386
Tasks: 4377615
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42984 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a4ba6455
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a4ba6455
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a4ba6455
Branch: refs/heads/0.89-fb
Commit: a4ba6455c981b328312e0444618e893b4d76f26f
Parents: b0dbf01
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 19 18:38:20 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
src/main/java/org/apache/hadoop/hbase/client/TableServers.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a4ba6455/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index a95e2ba..9be35d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -50,6 +50,7 @@ import java.util.concurrent.FutureTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.TimeoutException;
import javax.annotation.Nullable;
@@ -1154,6 +1155,7 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
e instanceof SyncFailedException ||
e instanceof EOFException ||
e instanceof TTransportException ||
+ e instanceof TimeoutException ||
isNetworkException(e.getCause()));
}
[49/50] [abbrv] git commit: Revert "[HBASE-11233] Asynchronous
multiputs"
Posted by ec...@apache.org.
Revert "[HBASE-11233] Asynchronous multiputs"
Summary: This reverts commit fb2b5adfb05266cb77619c1544d9d6463feabcf0.
Test Plan: simple revert.
Reviewers: rshroff
Reviewed By: rshroff
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1408564
Tasks: 4584691
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43193 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da748d43
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da748d43
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da748d43
Branch: refs/heads/0.89-fb
Commit: da748d43de8f67edabc7b5a12729099a2bef8fb6
Parents: 5808b33
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jun 27 19:07:43 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:52:44 2014 -0700
----------------------------------------------------------------------
.../hbase/master/TimeToLiveLogCleaner.java | 5 +-
.../hadoop/hbase/regionserver/HRegion.java | 266 ++++++-------------
.../hbase/regionserver/HRegionServer.java | 102 +++----
.../MultiVersionConsistencyControl.java | 3 +
.../hadoop/hbase/regionserver/wal/HLog.java | 222 ++++++----------
.../hadoop/hbase/HBaseTestingUtility.java | 4 -
.../hadoop/hbase/client/TestAsyncMultiputs.java | 185 -------------
.../org/apache/hadoop/hbase/client/TestHCM.java | 9 +-
.../hadoop/hbase/master/TestOldLogsCleaner.java | 5 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 9 +-
.../wal/TestAsyncMultiputsFailingWal.java | 58 ----
.../hadoop/hbase/regionserver/wal/TestHLog.java | 52 +---
.../hbase/regionserver/wal/TestHLogSplit.java | 6 +-
.../wal/TestLogActionsListener.java | 2 +-
.../hbase/regionserver/wal/TestWALReplay.java | 9 +-
15 files changed, 229 insertions(+), 708 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
index fce153e..cc9e571 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
@@ -48,9 +48,8 @@ public class TimeToLiveLogCleaner implements LogCleanerDelegate {
long currentTime = System.currentTimeMillis();
try {
// If the path name is in hourly format, skip getting modification time
- if (OldLogsCleaner.isOldLogsArchivedToHourlyDir()
- && OldLogsCleaner.isMatchDatePattern(filePath)) {
- time = HLog.getDateFormat().parse(filePath.getName()).getTime();
+ if (OldLogsCleaner.isOldLogsArchivedToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) {
+ time = HLog.DATE_FORMAT.parse(filePath.getName()).getTime();
} else {
FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath);
time = fStat.getModificationTime();
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4f7b42c..2511c8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -56,8 +56,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import javax.annotation.Nullable;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -108,14 +106,9 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.AsyncFunction;
-import com.google.common.util.concurrent.CheckedFuture;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
/**
* HRegion stores data for a certain region of a table. It stores all columns
@@ -1981,7 +1974,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
seqNum = this.log.append(regionInfo,
- regionInfo.getTableDesc().getName(), walEdit, now).checkedGet();
+ regionInfo.getTableDesc().getName(), walEdit, now);
}
// Now make changes to the memstore.
@@ -2062,7 +2055,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
try {
// All edits for the given row (across all column families) must happen atomically.
- put(put.getFamilyMap(), writeToWAL).checkedGet();
+ put(put.getFamilyMap(), writeToWAL);
} finally {
if (lockid == null) releaseRowLock(lid);
}
@@ -2097,28 +2090,14 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* Perform a batch put with no pre-specified locks
* @see HRegion#put(Pair[])
*/
- public CheckedFuture<OperationStatusCode[], IOException> put(Put[] puts) throws IOException {
+ public OperationStatusCode[] put(Put[] puts) throws IOException {
@SuppressWarnings("unchecked")
Pair<Mutation, Integer> putsAndLocks[] = new Pair[puts.length];
for (int i = 0; i < puts.length; i++) {
putsAndLocks[i] = new Pair<Mutation, Integer>(puts[i], null);
}
- return Futures.makeChecked(batchMutateWithLocks(putsAndLocks, "multiput_"),
- getExceptionMapperFunction());
- }
-
- public Function<Exception, IOException> getExceptionMapperFunction() {
- return new Function<Exception, IOException>() {
- @Override
- @Nullable
- public IOException apply(@Nullable Exception e) {
- if (e instanceof ExecutionException) {
- return new IOException(e.getCause());
- }
- return new IOException(e);
- }
- };
+ return batchMutateWithLocks(putsAndLocks, "multiput_");
}
/**
@@ -2127,55 +2106,33 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* @param methodName "multiput_/multidelete_" to update metrics correctly.
* @throws IOException
*/
- public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
- Pair<Mutation, Integer>[] putsAndLocks, String methodName)
- throws IOException {
+ public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
+ String methodName) throws IOException {
BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
new BatchOperationInProgress<Pair<Mutation,Integer>>(putsAndLocks);
- return batchMutateWithLazyRetry(batchOp, methodName);
- }
- public ListenableFuture<OperationStatusCode[]> batchMutateWithLazyRetry(
- final BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
- final String methodName) throws IOException {
-
- if (!batchOp.isDone()) {
+ while (!batchOp.isDone()) {
checkReadOnly();
checkResources();
+ long newSize;
splitsAndClosesLock.readLock().lock();
try {
doPreMutationHook(batchOp);
- return Futures.transform(doMiniBatchOp(batchOp, methodName),
- new AsyncFunction<Long, OperationStatusCode[]>() {
- @Override
- public ListenableFuture<OperationStatusCode[]> apply(Long addedSize)
- throws Exception {
- splitsAndClosesLock.readLock().lock();
- try {
- long newSize = incMemoryUsage(addedSize);
- if (isFlushSize(newSize)) {
- requestFlush();
- }
- } finally {
- splitsAndClosesLock.readLock().unlock();
- }
- if (batchOp.isDone()) {
- HRegion.writeOps.incrementAndGet();
- return Futures.immediateFuture(batchOp.retCodes);
- }
- return batchMutateWithLazyRetry(batchOp, methodName);
- }
- });
- } catch (Exception e) {
- return Futures.immediateFuture(batchOp.retCodes);
+ long addedSize = doMiniBatchOp(batchOp, methodName);
+ newSize = this.incMemoryUsage(addedSize);
} finally {
splitsAndClosesLock.readLock().unlock();
}
+ if (isFlushSize(newSize)) {
+ requestFlush();
+ }
}
- return Futures.immediateFuture(batchOp.retCodes);
+ HRegion.writeOps.incrementAndGet();
+ return batchOp.retCodes;
}
+
/**
* Execute coprocessor hooks for mutations
* TODO: adela currently can work only with prePut and we need to implement the rest..
@@ -2208,26 +2165,23 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
- private ListenableFuture<Long> doMiniBatchOp(
- final BatchOperationInProgress <Pair<Mutation, Integer>> batchOp,
- final String methodNameForMetricsUpdate)
- throws IOException {
- boolean preWallEditSuccessfull = false;
+ private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
+ String methodNameForMetricsUpdate) throws IOException {
String signature = null;
// variable to note if all Put items are for the same CF -- metrics related
boolean isSignatureClear = true;
- final long now = EnvironmentEdgeManager.currentTimeMillis();
+ long now = EnvironmentEdgeManager.currentTimeMillis();
byte[] byteNow = Bytes.toBytes(now);
boolean locked = false;
/** Keep track of the locks we hold so we can release them in finally clause */
- final List<Integer> acquiredLocks =
- Lists.newArrayListWithCapacity(batchOp.operations.length);
+ List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
// We try to set up a batch in the range [firstIndex,lastIndexExclusive)
- final int firstIndex = batchOp.nextIndexToProcess;
+ int firstIndex = batchOp.nextIndexToProcess;
int lastIndexExclusive = firstIndex;
+ boolean success = false;
try {
// ------------------------------------
// STEP 1. Try to acquire as many locks as we can, and ensure
@@ -2239,8 +2193,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
Integer currentLockID = null;
while (lastIndexExclusive < batchOp.operations.length) {
- Pair<Mutation, Integer> nextPair =
- batchOp.operations[lastIndexExclusive];
+ Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
Mutation op = nextPair.getFirst();
Integer providedLockId = nextPair.getSecond();
@@ -2291,9 +2244,6 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
}
-
- final int finalLastIndexExclusive = lastIndexExclusive;
-
// We've now grabbed as many puts off the list as we can
assert numReadyToWrite > 0;
@@ -2303,7 +2253,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// ------------------------------------
// STEP 2. Update any LATEST_TIMESTAMP timestamps
// ----------------------------------
- for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
+ for (int i = firstIndex; i < lastIndexExclusive; i++) {
Mutation op = batchOp.operations[i].getFirst();
if (op instanceof Put) {
@@ -2317,14 +2267,13 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
- ListenableFuture<Long> wallEditFuture =
- Futures.immediateFuture(-1L);
+ long seqNum = -1;
// ------------------------------------
// STEP 3. Write to WAL
// ----------------------------------
if (!this.disableWAL) {
WALEdit walEdit = new WALEdit();
- for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
+ for (int i = firstIndex; i < lastIndexExclusive; i++) {
// Skip puts that were determined to be invalid during preprocessing
if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
@@ -2334,80 +2283,48 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
// Append the edit to WAL
- wallEditFuture = this.log.append(regionInfo,
- regionInfo.getTableDesc().getName(),
- walEdit, now);
+ seqNum = this.log.append(regionInfo,
+ regionInfo.getTableDesc().getName(),
+ walEdit, now);
}
- final String finalSignature = signature;
-
- preWallEditSuccessfull = true;
-
- return Futures.transform(wallEditFuture, new AsyncFunction<Long, Long>() {
- @Override
- public ListenableFuture<Long> apply(Long input) throws Exception {
- boolean memstoreUpdatesSuccessful = false;
- updatesLock.readLock().lock();
- try {
- // ------------------------------------
- // STEP 4. Write back to memstore
- // ----------------------------------
-
- long addedSize = 0;
- for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
- if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
-
- Mutation op = batchOp.operations[i].getFirst();
- addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), input);
- batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
- }
- memstoreUpdatesSuccessful = true;
- return Futures.immediateFuture(addedSize);
- } finally {
- releaseLocksAndUpdateMetrics(updatesLock, acquiredLocks,
- finalSignature, methodNameForMetricsUpdate, firstIndex,
- finalLastIndexExclusive, batchOp, true,
- memstoreUpdatesSuccessful, now);
- }
- }
- });
- } catch (Throwable t) {
- return Futures.immediateFuture(null);
- } finally {
- releaseLocksAndUpdateMetrics(updatesLock,
- Collections.<Integer> emptyList(), signature,
- methodNameForMetricsUpdate, firstIndex, lastIndexExclusive, batchOp,
- locked, preWallEditSuccessfull, now);
- }
- }
- /**
- * This method was intended for code de-duplication and nothing else.
- */
- private void releaseLocksAndUpdateMetrics(ReentrantReadWriteLock updatedLock,
- List<Integer> acquiredLocks, String signature,
- String methodNameForMetricsUpdate, int firstIndex, int lastIndexExclusive,
- BatchOperationInProgress<Pair<Mutation, Integer>> batchOp, boolean locked,
- boolean preWallEditSuccessfull, long now) {
- if (locked) {
- this.updatesLock.readLock().unlock();
- }
- releaseRowLocks(acquiredLocks);
- // do after lock
- long after = EnvironmentEdgeManager.currentTimeMillis();
- if (null == signature) {
- signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
- }
- HRegion.incrTimeVaryingMetric(
- signature + methodNameForMetricsUpdate, after - now);
+ // ------------------------------------
+ // STEP 4. Write back to memstore
+ // ----------------------------------
- if (!preWallEditSuccessfull) {
+ long addedSize = 0;
for (int i = firstIndex; i < lastIndexExclusive; i++) {
- if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
- batchOp.retCodes[i] = OperationStatusCode.FAILURE;
+ if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
+
+ Mutation op = batchOp.operations[i].getFirst();
+ addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), seqNum);
+ batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
+ }
+ success = true;
+ return addedSize;
+ } finally {
+ if (locked) {
+ this.updatesLock.readLock().unlock();
+ }
+
+ releaseRowLocks(acquiredLocks);
+
+ // do after lock
+ long after = EnvironmentEdgeManager.currentTimeMillis();
+ if (null == signature) {
+ signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
+ }
+ HRegion.incrTimeVaryingMetric(signature + methodNameForMetricsUpdate, after - now);
+
+ if (!success) {
+ for (int i = firstIndex; i < lastIndexExclusive; i++) {
+ if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
+ batchOp.retCodes[i] = OperationStatusCode.FAILURE;
+ }
}
}
+ batchOp.nextIndexToProcess = lastIndexExclusive;
}
- batchOp.nextIndexToProcess = lastIndexExclusive;
}
//TODO, Think that gets/puts and deletes should be refactored a bit so that
@@ -2468,7 +2385,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
if (matches) {
// All edits for the given row (across all column families) must happen atomically.
if (isPut) {
- put(((Put)w).getFamilyMap(), writeToWAL).checkedGet();
+ put(((Put)w).getFamilyMap(), writeToWAL);
} else {
Delete d = (Delete)w;
prepareDeleteFamilyMap(d);
@@ -2564,7 +2481,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
throws IOException {
Map<byte[], List<KeyValue>> familyMap = new HashMap<byte[], List<KeyValue>>();
familyMap.put(family, edits);
- this.put(familyMap, true).checkedGet();
+ this.put(familyMap, true);
}
/**
@@ -2574,12 +2491,11 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* @param writeToWAL if true, then we should write to the log
* @throws IOException
*/
- private CheckedFuture<Void, IOException> put(final Map<byte [], List<KeyValue>> familyMap,
+ private void put(final Map<byte [], List<KeyValue>> familyMap,
boolean writeToWAL) throws IOException {
- final long now = EnvironmentEdgeManager.currentTimeMillis();
+ long now = EnvironmentEdgeManager.currentTimeMillis();
byte[] byteNow = Bytes.toBytes(now);
- ListenableFuture<Long> wallEditSyncFuture =
- Futures.immediateFuture(new Long(-1));
+ boolean flush = false;
this.updatesLock.readLock().lock();
try {
checkFamilies(familyMap.keySet());
@@ -2590,43 +2506,30 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// If order is reversed, i.e. we write to memstore first, and
// for some reason fail to write/sync to commit log, the memstore
// will contain uncommitted transactions.
+ long seqNum = -1;
if (!this.disableWAL && writeToWAL) {
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
- wallEditSyncFuture = this.log.append(regionInfo,
+ seqNum = this.log.append(regionInfo,
regionInfo.getTableDesc().getName(), walEdit, now);
}
- return Futures.makeChecked(Futures.transform(wallEditSyncFuture,
- new AsyncFunction<Long, Void>() {
- @Override
- public ListenableFuture<Void> apply(Long input) throws Exception {
- boolean flush = false;
- updatesLock.readLock().lock();
- try {
- long addedSize = applyFamilyMapToMemstore(familyMap, input);
- flush = isFlushSize(incMemoryUsage(addedSize));
- } finally {
- updatesLock.readLock().unlock();
- }
- // do after lock
- long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = SchemaMetrics.generateSchemaMetricsPrefix(
- getTableDesc().getNameAsString(), familyMap.keySet());
- HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
-
- if (flush) {
- // Request a cache flush. Do it outside update lock.
- requestFlush();
- }
- return Futures.immediateFuture(null);
- }
- }), this.getExceptionMapperFunction());
- } catch (Exception e) {
- return Futures.immediateFailedCheckedFuture(new IOException(e));
+ long addedSize = applyFamilyMapToMemstore(familyMap, seqNum);
+ flush = isFlushSize(this.incMemoryUsage(addedSize));
} finally {
this.updatesLock.readLock().unlock();
}
+
+ // do after lock
+ long after = EnvironmentEdgeManager.currentTimeMillis();
+ String signature = SchemaMetrics.generateSchemaMetricsPrefix(
+ this.getTableDesc().getNameAsString(), familyMap.keySet());
+ HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
+
+ if (flush) {
+ // Request a cache flush. Do it outside update lock.
+ requestFlush();
+ }
}
/**
@@ -3885,9 +3788,8 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// 6. append/sync all edits at once
// TODO: Do batching as in doMiniBatchPut
- long seqNum;
- seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
- walEdit, now).checkedGet();
+ long seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
+ walEdit, now);
// 7. apply to memstore
long addedSize = 0;
@@ -3973,7 +3875,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
WALEdit walEdit = new WALEdit();
walEdit.add(newKv);
seqNum = this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now).checkedGet();
+ walEdit, now);
}
// Now request the ICV to the store, this will set the timestamp
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 207760f..a742a29 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -64,8 +64,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import javax.annotation.Nullable;
-
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
@@ -176,12 +174,7 @@ import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.AsyncFunction;
-import com.google.common.util.concurrent.CheckedFuture;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
@@ -2119,13 +2112,6 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
return this.hlogs[i];
}
- /**
- * Return number of logs present in the regionserver
- */
- public int getLogCount() {
- return this.hlogs.length;
- }
-
public int getTotalHLogCnt() {
return this.hlogs.length;
}
@@ -2979,18 +2965,13 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
}
}
- public CheckedFuture<Integer, IOException> putAsync(final byte[] regionName,
- final List<Put> puts) throws IOException {
- return applyMutations(regionName, puts, "multiput_");
- }
-
@Override
- public int put(final byte[] regionName,
- final List<Put> puts) throws IOException {
- return applyMutations(regionName, puts, "multiput_").checkedGet();
+ public int put(final byte[] regionName, final List<Put> puts)
+ throws IOException {
+ return applyMutations(regionName, puts, "multiput_");
}
- private CheckedFuture<Integer, IOException> applyMutations(final byte[] regionName,
+ private int applyMutations(final byte[] regionName,
final List<? extends Mutation> mutations,
String methodName)
throws IOException {
@@ -3011,45 +2992,18 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
opWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
}
- ListenableFuture<OperationStatusCode[]> batchMutateFuture =
- region.batchMutateWithLocks(opWithLocks,
+ OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks,
methodName);
- return Futures.makeChecked(Futures.transform(batchMutateFuture,
- new AsyncFunction<OperationStatusCode[], Integer>() {
-
- @Override
- public ListenableFuture<Integer> apply(OperationStatusCode[] codes)
- throws Exception {
- try {
- for (int i = 0; i < codes.length; i++) {
- if (codes[i] != OperationStatusCode.SUCCESS)
- return Futures.immediateFuture(i);
- }
- return Futures.immediateFuture(HConstants.MULTIPUT_SUCCESS);
- } catch (Throwable t) {
- return Futures.immediateFailedCheckedFuture(
- convertThrowableToIOE(cleanup(t)));
- }
- }
- }), getConvertThrowableToIOEFunction());
+ for (i = 0; i < codes.length; i++) {
+ if (codes[i] != OperationStatusCode.SUCCESS)
+ return i;
+ }
+ return HConstants.MULTIPUT_SUCCESS;
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
}
}
- private Function<Exception, IOException> getConvertThrowableToIOEFunction() {
- return new Function<Exception, IOException>() {
- @Override
- @Nullable
- public IOException apply(@Nullable Exception e) {
- if (e instanceof ExecutionException) {
- return convertThrowableToIOE(cleanup(e.getCause()));
- }
- return convertThrowableToIOE(cleanup(e));
- }
- };
- }
-
private boolean checkAndMutate(final byte[] regionName, final byte [] row,
final byte [] family, final byte [] qualifier, final byte [] value,
final Writable w, Integer lock) throws IOException {
@@ -3312,7 +3266,7 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
public int delete(final byte[] regionName,
final List<Delete> deletes)
throws IOException {
- return applyMutations(regionName, deletes, "multidelete_").checkedGet();
+ return applyMutations(regionName, deletes, "multidelete_");
}
@Override
@@ -3961,18 +3915,32 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
public MultiPutResponse multiPut(MultiPut puts) throws IOException {
MultiPutResponse resp = new MultiPutResponse();
- Map<byte[], CheckedFuture<Integer, IOException>> futuresMap =
- new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ // do each region as it's own.
+ int size = puts.puts.size();
+ int index = 0;
for( Map.Entry<byte[], List<Put>> e: puts.puts.entrySet()) {
- futuresMap.put(e.getKey(), putAsync(e.getKey(), e.getValue()));
- }
- for (Entry<byte[], CheckedFuture<Integer, IOException>> e :
- futuresMap.entrySet()) {
- CheckedFuture<Integer, IOException> f = e.getValue();
- byte[] row = e.getKey();
- int ret = f.checkedGet();
- resp.addResult(row, ret);
+ int result = put(e.getKey(), e.getValue());
+ resp.addResult(e.getKey(), result);
+
+ index++;
+ if (index < size) {
+ // remove the reference to the region list of Puts to save RAM except
+ // for the last one. We will lose the reference to the last one pretty
+ // soon anyway; keep it for a little more, until we get back
+ // to HBaseServer level, where we might need to pretty print
+ // the MultiPut request for debugging slow/large puts.
+ // Note: A single row "put" from client also end up in server
+ // as a multiPut().
+ // We set the value to an empty array so taskmonitor can either get
+ // the old value or an empty array. If we call clear() on the array,
+ // then we might have a race condition where we're iterating over the
+ // array at the same time we clear it (which throws an exception)
+ // This relies on the fact that Map.Entry.setValue() boils down to a
+ // simple reference assignment, which is atomic
+ e.setValue(emptyPutArray); // clear some RAM
+ }
}
+
return resp;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
index 53e9757..6b28f03 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
@@ -24,6 +24,9 @@ import java.util.LinkedList;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+
/**
* Manages the read/write consistency within memstore. This provides
* an interface for readers to determine what entries to ignore, and
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index e7aa701..0cfaa8f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -63,8 +63,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.annotation.Nullable;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -82,10 +80,13 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.ipc.HBaseServer.Call;
+import org.apache.hadoop.hbase.ipc.ProfilingData;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionSeqidTransition;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -98,13 +99,7 @@ import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.AsyncFunction;
-import com.google.common.util.concurrent.CheckedFuture;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -151,9 +146,7 @@ public class HLog implements Syncable {
public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
public static final byte [] METAROW = Bytes.toBytes("METAROW");
public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
- public static SimpleDateFormat getDateFormat() {
- return new SimpleDateFormat("yyyy-MM-dd-HH");
- }
+ public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH");
/*
* Name of directory that holds recovered edits written by the wal log
@@ -188,8 +181,8 @@ public class HLog implements Syncable {
Collections.synchronizedList(new ArrayList<LogActionsListener>());
- volatile static Class<? extends Writer> logWriterClass;
- volatile static Class<? extends Reader> logReaderClass;
+ static Class<? extends Writer> logWriterClass;
+ static Class<? extends Reader> logReaderClass;
private OutputStream hdfs_out; // OutputStream associated with the current SequenceFile.writer
private int initialReplication; // initial replication factor of SequenceFile.writer
@@ -291,6 +284,9 @@ public class HLog implements Syncable {
private final List<LogEntryVisitor> logEntryVisitors =
new CopyOnWriteArrayList<LogEntryVisitor>();
+ private volatile long lastLogRollStartTimeMillis = 0;
+ private volatile long lastLogRollDurationMillis = 0;
+
/**
* Pattern used to validate a HLog file name
*/
@@ -309,9 +305,9 @@ public class HLog implements Syncable {
synchronized (this) {
min = Math.min(min, val);
max = Math.max(max, val);
- total += val;
- ++count;
}
+ total += val;
+ ++count;
}
Metric get() {
@@ -363,17 +359,14 @@ public class HLog implements Syncable {
*/
private class DoubleListBuffer {
private LinkedList<Entry> currentList = new LinkedList<Entry>();
- private SettableFuture<Void> currentFuture = SettableFuture.create();;
private LinkedList<Entry> syncList = new LinkedList<Entry>();
/**
* Append a log entry into the buffer
* @param entry log entry
*/
- synchronized private ListenableFuture<Void> appendToBuffer(Entry entry) {
- Preconditions.checkNotNull(this.currentFuture);
+ synchronized private void appendToBuffer(Entry entry) {
currentList.add(entry);
- return this.currentFuture;
}
/**
@@ -382,8 +375,6 @@ public class HLog implements Syncable {
* @return number of log entries synced
*/
private int appendAndSync() throws IOException {
- // The future returned to the append operation.
- SettableFuture<Void> future;
synchronized (this) {
if (currentList.isEmpty()) { // no thing to sync
return 0;
@@ -394,49 +385,26 @@ public class HLog implements Syncable {
LinkedList<Entry> tmp = syncList;
syncList = currentList;
currentList = tmp;
- future = currentFuture;
- this.currentFuture = SettableFuture.create();
}
- try {
- // append entries to writer
- int syncedEntries = syncList.size();
- while (!syncList.isEmpty()) {
- Entry entry = syncList.remove();
- writer.append(entry);
- }
-
- // sync the data
- long now = System.currentTimeMillis();
- writer.sync();
- syncTime.inc(System.currentTimeMillis() - now);
- future.set(null);
- return syncedEntries;
- } catch (Exception e) {
- future.setException(e);
- return 0;
+ // append entries to writer
+ int syncedEntries = syncList.size();
+ while (!syncList.isEmpty()) {
+ Entry entry = syncList.remove();
+ writer.append(entry);
}
+
+ // sync the data
+ long now = System.currentTimeMillis();
+ writer.sync();
+ syncTime.inc(System.currentTimeMillis() - now);
+ return syncedEntries;
}
}
private DoubleListBuffer logBuffer = new DoubleListBuffer();
/**
- * Minimal constructor to get going in tests
- */
- public HLog(final Configuration conf, Path basePath) throws IOException {
- this(FileSystem.get(conf), new Path(basePath, ".logs"),
- new Path(basePath, ".oldlogs"), conf, null);
- }
-
- /**
- * Minimal constructor to get going in tests
- */
- public HLog(final Configuration conf) throws IOException {
- this(conf, new Path("/tmp"));
- }
-
- /**
* HLog creating with a null actions listener.
*
* @param fs filesystem handle
@@ -683,6 +651,8 @@ public class HLog implements Syncable {
this.numEntries.set(0);
t1 = EnvironmentEdgeManager.currentTimeMillis();
+ lastLogRollStartTimeMillis = t0;
+ lastLogRollDurationMillis = (t1 - t0);
}
Path oldFile = null;
@@ -729,7 +699,7 @@ public class HLog implements Syncable {
throws IOException {
try {
if (logReaderClass == null) {
- logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
+ logReaderClass =conf.getClass("hbase.regionserver.hlog.reader.impl",
SequenceFileLogReader.class, Reader.class);
}
@@ -738,7 +708,8 @@ public class HLog implements Syncable {
return reader;
} catch (IOException e) {
throw e;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
throw new IOException("Cannot get log reader", e);
}
}
@@ -1081,10 +1052,10 @@ public class HLog implements Syncable {
* of the memstore.
* @throws IOException
*/
- public CheckedFuture<Long, IOException> append(HRegionInfo info, byte[] tableName, WALEdit edits,
+ public long append(HRegionInfo info, byte [] tableName, WALEdit edits,
final long now) throws IOException {
if (!this.enabled || edits.isEmpty()) {
- return Futures.immediateCheckedFuture(-1L);
+ return -1L;
}
if (logSyncerThread.syncerShuttingDown) {
// can't acquire lock for the duration of append()
@@ -1092,13 +1063,13 @@ public class HLog implements Syncable {
throw new IOException("Cannot append; logSyncer shutting down");
}
- final long len = edits.getTotalKeyValueLength();
- final long txid;
- final long seqNum;
+ long len = edits.getTotalKeyValueLength();
+ long txid = 0;
+ long seqNum;
+ long start = System.currentTimeMillis();
byte[] regionName = info.getRegionName();
- ListenableFuture<Void> logSyncFuture;
synchronized (this.appendLock) {
// The 'lastSeqWritten' map holds the sequence number of the oldest
// write for each region (i.e. the first edit added to the particular
@@ -1109,54 +1080,50 @@ public class HLog implements Syncable {
this.firstSeqWrittenInCurrentMemstore.putIfAbsent(regionName, seqNum);
HLogKey logKey = makeKey(regionName, tableName, seqNum, now);
- logSyncFuture = doWrite(info, logKey, edits);
+ doWrite(info, logKey, edits);
// Only count 1 row as an unflushed entry.
txid = this.unflushedEntries.incrementAndGet();
}
- if (info.isMetaRegion()) forceSync();
- final long start = System.nanoTime();
+ // Update the metrics
+ this.numEntries.incrementAndGet();
+ writeSize.inc(len);
- return Futures.makeChecked(Futures.transform(logSyncFuture,
- new AsyncFunction<Void, Long>() {
- @Override
- public ListenableFuture<Long> apply(Void input) throws Exception {
- Preconditions.checkArgument(txid >= getCurSyncPoint());
- // Update the metrics
- numEntries.incrementAndGet();
- writeSize.inc(len);
-
- // Update the metrics and log down the outliers
- long end = System.nanoTime();
- long syncTime = end - start;
- gsyncTime.inc(syncTime);
-
- if (syncTime > SECOND_IN_NS) {
- LOG.warn(String.format(
- "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
- Thread.currentThread().getName(), syncTime, numEntries.get(),
- StringUtils.humanReadableInt(len)));
- slowSyncs.incrementAndGet();
- }
- if (slowSyncs.get() >= slowBeforeRoll) {
- requestLogRoll();
- slowSyncs.set(0);
- }
- return Futures.immediateFuture(new Long(seqNum));
- }
- }), this.getExceptionMapperFunction());
- }
+ // sync txn to file system
+ start = System.nanoTime();
+ this.sync(info.isMetaRegion(), txid);
- public Function<Exception, IOException> getExceptionMapperFunction() {
- return new Function<Exception, IOException>() {
- @Override
- @Nullable
- public IOException apply(@Nullable Exception e) {
- if (e instanceof ExecutionException) {
- return new IOException(e.getCause());
- }
- return new IOException(e);
+ // Update the metrics and log down the outliers
+ long end = System.nanoTime();
+ long syncTime = end - start;
+ gsyncTime.inc(syncTime);
+ if (syncTime > SECOND_IN_NS) {
+ LOG.warn(String.format(
+ "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
+ Thread.currentThread().getName(), syncTime, this.numEntries.get(),
+ StringUtils.humanReadableInt(len)));
+ slowSyncs.incrementAndGet();
+ }
+
+ if (slowSyncs.get() >= slowBeforeRoll) {
+ this.requestLogRoll();
+ this.slowSyncs.set(0);
+ }
+
+ // Update the per-request profiling data
+ Call call = HRegionServer.callContext.get();
+ ProfilingData pData = call == null ? null : call.getProfilingData();
+ if (pData != null) {
+ if (this.lastLogRollStartTimeMillis > start
+ && end > this.lastLogRollStartTimeMillis) {
+ // We also had a log roll in between
+ pData.addLong(ProfilingData.HLOG_ROLL_TIME_MS, this.lastLogRollDurationMillis);
+ // Do not account for this as the sync time.
+ syncTime = syncTime - this.lastLogRollDurationMillis;
}
- };
+ // update sync time
+ pData.addLong(ProfilingData.HLOG_SYNC_TIME_MS, syncTime);
+ }
+ return seqNum;
}
/**
@@ -1199,7 +1166,7 @@ public class HLog implements Syncable {
forceSync = false;
// wake up every 100ms to check if logsyncer has to shut down
queueEmpty.awaitNanos(100*1000000);
- if (unflushedEntries.get() >= syncTillHere) {
+ if (unflushedEntries.get() == syncTillHere) {
// call hflush() if we haven't flushed for a while now
// This force-sync is just a safety mechanism - we being
// paranoid. If there hasn't been any syncing activity for
@@ -1242,18 +1209,6 @@ public class HLog implements Syncable {
}
/**
- * This method needs to be called to indicate that we need to force the
- * log syncer thread to sync.
- * @param force
- */
- public void setForceSync(boolean force) {
- forceSync = false;
- }
-
- public long getCurSyncPoint() {
- return syncTillHere;
- }
- /**
* This method first signals the thread that there's a sync needed
* and then waits for it to happen before returning.
* @throws IOException
@@ -1320,14 +1275,6 @@ public class HLog implements Syncable {
logSyncerThread.addToSyncQueue(force, txid);
}
- public void forceSync() {
- logSyncerThread.setForceSync(true);
- }
-
- public long getCurSyncPoint() {
- return logSyncerThread.getCurSyncPoint();
- }
-
private void hflush() {
synchronized (this.updateLock) {
if (this.closed) {
@@ -1431,9 +1378,9 @@ public class HLog implements Syncable {
}
}
- protected ListenableFuture<Void> doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
+ protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
throws IOException {
- return this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
+ this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
}
/** @return How many items have been added to the log */
@@ -1942,8 +1889,7 @@ public class HLog implements Syncable {
String subDirectoryName;
if (ARCHIVE_TO_HOURLY_DIR) {
// Group into hourly sub-directory
- subDirectoryName =
- getDateFormat().format(Calendar.getInstance().getTime());
+ subDirectoryName = DATE_FORMAT.format(Calendar.getInstance().getTime());
} else {
// since the filename is a valid name, we know there
// is a last '.' (won't return -1)
@@ -2441,25 +2387,11 @@ public class HLog implements Syncable {
walEdit.add(kvLast);
walEdit.add(kvNext);
walEdit.add(kvRegionInfo);
- long newSeqid;
- try {
- newSeqid = append(regionInfo, regionInfo.getTableDesc()
- .getName(), walEdit, now).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new IOException(e);
- }
+ long newSeqid = append(regionInfo, regionInfo.getTableDesc().getName(), walEdit, now);
LOG.info("Region " + regionInfo.getRegionNameAsString() +
" sequence id transition " +
seqidTransition.getLastSeqid() + "->" +
seqidTransition.getNextSeqid() + " appended to HLog at seqid=" + newSeqid);
return true;
}
-
- /**
- * Returns the underlying configuration object.
- * @return
- */
- public Configuration getConfiguration() {
- return this.conf;
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index b161f51..1b62945 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2093,8 +2093,4 @@ public class HBaseTestingUtility {
getHBaseAdmin().moveRegion(regionInfo.getRegionName(),
regionServer.toString());
}
-
- public HConnection getConnection() {
- return HConnectionManager.getConnection(this.conf);
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java b/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
deleted file mode 100644
index 3b0ef2b..0000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
+++ /dev/null
@@ -1,185 +0,0 @@
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.StringBytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestAsyncMultiputs {
- private static final Log LOG = LogFactory.getLog(TestAsyncMultiputs.class);
- private static final HBaseTestingUtility util = new HBaseTestingUtility();
- private static int NUM_REGIONSERVERS = 1;
-
- @BeforeClass
- public static void setUp() throws Exception {
- util.startMiniCluster(NUM_REGIONSERVERS);
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- util.shutdownMiniCluster();
- }
-
- /**
- * Test intended to check if the wal gives the control
- * back when writers are calling append.
- * @throws IOException
- * @throws InterruptedException
- */
- @Test
- public void testMultithreadedCallbackFromHLog() throws IOException, InterruptedException {
- final HLog log = new HLog(util.getConfiguration(),
- new Path("/tmp/testMultithreadedCallback"));
- int numThreads = 100;
- ExecutorService executor = Executors.newCachedThreadPool();
- for (int i = 0; i < numThreads; i++) {
- final int threadId = i;
- executor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- HTableDescriptor desc =
- new HTableDescriptor(Bytes.toBytes("table" + threadId));
- HRegionInfo info = new HRegionInfo(desc, new byte[0], new byte[0]);
- WALEdit edits = new WALEdit();
- for (byte[] row : getRandomRows(1, 10)) {
- edits.add(new KeyValue(row));
- }
- try {
- log.append(info, desc.getName(), edits,
- System.currentTimeMillis()).get(1, TimeUnit.MINUTES);
- } catch(Exception e) {
- fail();
- }
- return null;
- }
- });
- }
- executor.shutdown();
- executor.awaitTermination(1, TimeUnit.MINUTES);
- }
-
- @Test
- public void testMultiputSucceeds() throws IOException {
- String tableName = "testMultiputSucceeds";
- byte[] tableNameBytes = Bytes.toBytes(tableName);
- String cf = "cf";
- byte[][] families = new byte[][] { Bytes.toBytes(cf) };
- LOG.debug("Creating the table " + tableName);
- int numRegions = 100;
- HRegionServer server = createTableAndGiveMeTheRegionServer(tableNameBytes,
- families, numRegions);
- MultiPut multiput = getRandomMultiPut(tableNameBytes, 1000, families[0],
- server.getServerInfo().getServerAddress());
- server.multiPut(multiput);
- for (HRegion region : server.getOnlineRegionsAsArray()) {
- server.flushRegion(region.getRegionName());
- }
- for (Entry<byte[], List<Put>> entry : multiput.getPuts().entrySet()) {
- byte[] regionName = entry.getKey();
- for (Put p : entry.getValue()) {
- Get g = new Get(p.getRow());
- Result r = server.get(regionName, g);
- assertNotNull(g);
- assertTrue(Bytes.equals(r.getValue(families[0], null), p.getRow()));
- assertTrue(Bytes.equals(r.getRow(), p.getRow()));
- }
- }
- }
-
- public HRegionServer createTableAndGiveMeTheRegionServer(
- byte[] tableName, byte[][] families, int numregions) throws IOException {
- LOG.debug("Creating table " + Bytes.toString(tableName) + " with "
- + numregions + " regions");
- util.createTable(tableName, families, 1, Bytes.toBytes("aaa"),
- Bytes.toBytes("zzz"), numregions);
- assertTrue(NUM_REGIONSERVERS == 1);
- HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
- LOG.debug("Found the regionserver. Returning" + rs);
- return rs;
- }
-
- public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
- HServerAddress addr) throws IOException {
- return getRandomMultiPut(tableName, numPuts, cf, addr, util);
- }
-
- /**
- * Creates random puts with rows between aaa.. to zzz..
- * @param tableName
- * @param numPuts
- * @param cf
- * @param addr
- * @param util : The hbase testing utility to use.
- * @return
- * @throws IOException
- */
- public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
- HServerAddress addr, HBaseTestingUtility util) throws IOException {
- MultiPut multiput = new MultiPut(addr);
- List<Put> puts = getRandomPuts(numPuts, cf);
- LOG.debug("Creating " + numPuts + " puts to insert into the multiput");
- HConnection conn = util.getConnection();
- for (Put put : puts) {
- byte[] regionName = conn.getRegionLocation(new StringBytes(tableName),
- put.getRow(), false).getRegionInfo().getRegionName();
- multiput.add(regionName, put);
- }
- return multiput;
- }
-
- /**
- * Creates puts with rows between aaa... and zzz..
- * @param numPuts
- * @param cf
- * @return
- */
- public static List<Put> getRandomPuts(int numPuts, byte[] cf) {
- List<Put> ret = new ArrayList<Put> (numPuts);
- List<byte[]> rows = getRandomRows(numPuts, 3);
- for (byte[] row : rows) {
- Put p = new Put(row);
- p.add(cf, null, row);
- ret.add(p);
- }
- return ret;
- }
-
- public static List<byte[]> getRandomRows(int numRows, int rowlen) {
- Random rand = new Random();
- List<byte[]> ret = new ArrayList<byte[]>(numRows);
- for (int i = 0; i < numRows; i++) {
- byte[] row = new byte[rowlen];
- for (int j = 0; j < rowlen; j++) {
- row[j] = (byte) ('a' + (byte) rand.nextInt('z' - 'a'));
- }
- ret.add(row);
- }
- return ret;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 3f0c5bb..f22f256 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -50,9 +50,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-
/**
* This class is for testing HCM features
*/
@@ -278,11 +275,9 @@ public class TestHCM {
}
@Override
- public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
- Pair<Mutation, Integer>[] putsAndLocks,
+ public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
String methodName) throws IOException {
- return Futures.immediateFailedFuture(
- new IOException("Test induced failure"));
+ throw new IOException("Test induced failure");
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
index 758ae4e..42b7880 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
@@ -144,11 +144,10 @@ public class TestOldLogsCleaner {
fs.mkdirs(legacyDir);
fs.createNewFile(new Path(legacyDir, "123.456"));
Calendar cal = Calendar.getInstance();
- System.out.println("Now is: " + HLog.getDateFormat().format(cal.getTime()));
+ System.out.println("Now is: " + HLog.DATE_FORMAT.format(cal.getTime()));
for (int i = 0; i < 10; i++) {
cal.add(Calendar.HOUR, -1);
- Path hourDir = new Path(oldLogDir,
- HLog.getDateFormat().format(cal.getTime()));
+ Path hourDir = new Path(oldLogDir, HLog.DATE_FORMAT.format(cal.getTime()));
fs.mkdirs(hourDir);
fs.createNewFile(new Path(hourDir, new Path(fakeMachineName + "." + i)));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index cb4a9ec..437c19c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -372,7 +372,7 @@ public class TestHRegion extends HBaseTestCase {
puts[i].add(cf, qual, val);
}
- OperationStatusCode[] codes = this.region.put(puts).get();
+ OperationStatusCode[] codes = this.region.put(puts);
assertEquals(10, codes.length);
for (int i = 0; i < 10; i++) {
assertEquals(OperationStatusCode.SUCCESS, codes[i]);
@@ -382,7 +382,7 @@ public class TestHRegion extends HBaseTestCase {
LOG.info("Next a batch put with one invalid family");
puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
try {
- codes = this.region.put(puts).get();
+ codes = this.region.put(puts);
// put should throw an exception
assertTrue(false);
@@ -403,7 +403,7 @@ public class TestHRegion extends HBaseTestCase {
TestThread putter = new TestThread(ctx) {
@Override
public void doWork() throws IOException {
- retFromThread.set(region.put(puts).checkedGet());
+ retFromThread.set(region.put(puts));
}
};
LOG.info("...starting put thread while holding lock");
@@ -439,8 +439,7 @@ public class TestHRegion extends HBaseTestCase {
putsAndLocks.add(pair);
}
- codes = region.batchMutateWithLocks(putsAndLocks.toArray(
- new Pair[0]), "multiput_").get();
+ codes = region.batchMutateWithLocks(putsAndLocks.toArray(new Pair[0]), "multiput_");
LOG.info("...performed put");
for (int i = 0; i < 10; i++) {
assertEquals(OperationStatusCode.SUCCESS, codes[i]);
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
deleted file mode 100644
index 20150c0..0000000
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.hadoop.hbase.regionserver.wal;
-
-import static org.junit.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.MultiPut;
-import org.apache.hadoop.hbase.client.TestAsyncMultiputs;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.wal.TestHLog.FailingSequenceFileLogWriter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-public class TestAsyncMultiputsFailingWal {
- private final Log LOG = LogFactory.getLog(TestAsyncMultiputsFailingWal.class);
-
- @Test
- public void testFailedMultiput() throws Exception {
- String tableName = "testFailedMultiput";
- byte[] tableNameBytes = Bytes.toBytes(tableName);
- String cf = "cf";
- byte[][] families = new byte[][] { Bytes.toBytes(cf) };
- LOG.debug("Creating the table " + tableName);
- int numRegions = 3;
- HBaseTestingUtility localutil = new HBaseTestingUtility();
- localutil.startMiniCluster();
-
- LOG.debug("Creating table " + tableName + " with "
- + numRegions + " regions");
- localutil.createTable(tableNameBytes, families, 1, Bytes.toBytes("aaa"),
- Bytes.toBytes("zzz"), numRegions);
- HRegionServer rs = localutil.getRSForFirstRegionInTable(tableNameBytes);
- LOG.debug("Found the regionserver. Returning" + rs);
-
- HLog.logWriterClass = null;
- assertTrue(rs.getLogCount() == 1);
- HLog log = rs.getLog(0);
- log.getConfiguration().set("hbase.regionserver.hlog.writer.impl",
- FailingSequenceFileLogWriter.class.getName());
- log.rollWriter();
- MultiPut multiput = TestAsyncMultiputs.getRandomMultiPut(
- tableNameBytes, 1000, families[0],
- rs.getServerInfo().getServerAddress(), localutil);
-
- try {
- FailingSequenceFileLogWriter.setFailureMode(true);
- rs.multiPut(multiput);
- FailingSequenceFileLogWriter.setFailureMode(false);
- } catch (Exception e) {
- LOG.debug("Excepted to see an exception", e);
- return;
- }
- fail();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 59e666a..b7d644e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.ExecutionException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -139,12 +139,9 @@ public class TestHLog {
* Just write multiple logs then split. Before fix for HADOOP-2283, this
* would fail.
* @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
*/
@Test
- public void testSplit() throws IOException,
- InterruptedException, ExecutionException {
+ public void testSplit() throws IOException {
final byte [] tableName = Bytes.toBytes(TestHLog.class.getSimpleName());
final byte [] rowName = tableName;
@@ -170,7 +167,7 @@ public class TestHLog {
System.currentTimeMillis(), column));
LOG.info("Region " + i + ": " + edit);
log.append(infos[i], tableName, edit,
- System.currentTimeMillis()).get();
+ System.currentTimeMillis());
}
}
log.rollWriter();
@@ -219,7 +216,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
- wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
+ wal.append(info, bytes, kvs, System.currentTimeMillis());
}
// Now call sync and try reading. Opening a Reader before you sync just
// gives you EOFE.
@@ -237,7 +234,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
- wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
+ wal.append(info, bytes, kvs, System.currentTimeMillis());
}
reader = HLog.getReader(fs, walPath, conf);
count = 0;
@@ -256,7 +253,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, value));
- wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
+ wal.append(info, bytes, kvs, System.currentTimeMillis());
}
// Now I should have written out lots of blocks. Sync then read.
wal.sync();
@@ -353,7 +350,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
- wal.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
+ wal.append(regioninfo, tableName, kvs, System.currentTimeMillis());
}
// Now call sync to send the data to HDFS datanodes
wal.sync(true);
@@ -451,12 +448,9 @@ public class TestHLog {
/**
* Tests that we can write out an edit, close, and then read it back in again.
* @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
*/
@Test
- public void testEditAdd() throws IOException,
- InterruptedException, ExecutionException {
+ public void testEditAdd() throws IOException {
final int COL_COUNT = 10;
final byte [] tableName = Bytes.toBytes("tablename");
final byte [] row = Bytes.toBytes("row");
@@ -475,7 +469,7 @@ public class TestHLog {
HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName),
row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
final byte [] regionName = info.getRegionName();
- log.append(info, tableName, cols, System.currentTimeMillis()).get();
+ log.append(info, tableName, cols, System.currentTimeMillis());
long logSeqId = log.startCacheFlush(info.getRegionName());
log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion());
log.close();
@@ -547,12 +541,9 @@ public class TestHLog {
/**
* @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
*/
@Test
- public void testAppend() throws IOException,
- InterruptedException, ExecutionException {
+ public void testAppend() throws IOException {
final int COL_COUNT = 10;
final byte [] tableName = Bytes.toBytes("tablename");
final byte [] row = Bytes.toBytes("row");
@@ -570,7 +561,7 @@ public class TestHLog {
}
HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
- log.append(hri, tableName, cols, System.currentTimeMillis()).get();
+ log.append(hri, tableName, cols, System.currentTimeMillis());
long logSeqId = log.startCacheFlush(hri.getRegionName());
log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false);
log.close();
@@ -650,13 +641,13 @@ public class TestHLog {
}
private void addEdits(HLog log, HRegionInfo hri, byte [] tableName,
- int times) throws IOException, InterruptedException, ExecutionException {
+ int times) throws IOException {
final byte [] row = Bytes.toBytes("row");
for (int i = 0; i < times; i++) {
long timestamp = System.currentTimeMillis();
WALEdit cols = new WALEdit();
cols.add(new KeyValue(row, row, row, timestamp, row));
- log.append(hri, tableName, cols, timestamp).checkedGet();
+ log.append(hri, tableName, cols, timestamp);
}
}
@@ -673,23 +664,6 @@ public class TestHLog {
}
}
- public static class FailingSequenceFileLogWriter
- extends SequenceFileLogWriter implements HLog.Writer {
- static boolean enabled = false;
- public synchronized static void setFailureMode(boolean enabled) {
- FailingSequenceFileLogWriter.enabled = enabled;
- }
-
- @Override
- public void sync() throws IOException {
- if (FailingSequenceFileLogWriter.enabled) {
- throw new IOException(
- "Failing here to see if the error propogates upstream");
- }
- super.sync();
- }
- }
-
static class DumbLogEntriesVisitor implements LogEntryVisitor {
int increments = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 8d7b3b1..6a5e4a2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -52,7 +52,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -541,8 +540,7 @@ public class TestHLogSplit {
* safely commence on the master.
* */
@Test
- public void testLogRollAfterSplitStart() throws IOException,
- InterruptedException, ExecutionException {
+ public void testLogRollAfterSplitStart() throws IOException {
// set flush interval to a large number so it doesn't interrupt us
final String F_INTERVAL = "hbase.regionserver.optionallogflushinterval";
long oldFlushInterval = conf.getLong(F_INTERVAL, 1000);
@@ -563,7 +561,7 @@ public class TestHLogSplit {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
- log.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
+ log.append(regioninfo, tableName, kvs, System.currentTimeMillis());
}
// Send the data to HDFS datanodes and close the HDFS writer
log.sync(true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
index 2297b40..d1f6890 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
@@ -97,7 +97,7 @@ public class TestLogActionsListener {
KeyValue kv = new KeyValue(b,b,b);
WALEdit edit = new WALEdit();
edit.add(kv);
- hlog.append(hri, b, edit, 0).get();
+ hlog.append(hri, b, edit, 0);
if (i == 10) {
hlog.addLogActionsListerner(laterList);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da748d43/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index b8571c4..62e954f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -433,14 +432,14 @@ public class TestWALReplay {
long now = ee.currentTimeMillis();
edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
now, rowName));
- wal.append(hri, tableName, edit, now).get();
+ wal.append(hri, tableName, edit, now);
// Delete the c family to verify deletes make it over.
edit = new WALEdit();
now = ee.currentTimeMillis();
edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
KeyValue.Type.DeleteFamily));
- wal.append(hri, tableName, edit, now).get();
+ wal.append(hri, tableName, edit, now);
// Sync.
wal.sync();
@@ -580,7 +579,7 @@ public class TestWALReplay {
private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
final byte [] rowName, final byte [] family,
final int count, EnvironmentEdge ee, final HLog wal)
- throws IOException, InterruptedException, ExecutionException {
+ throws IOException {
String familyStr = Bytes.toString(family);
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
@@ -588,7 +587,7 @@ public class TestWALReplay {
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes,
ee.currentTimeMillis(), columnBytes));
- wal.append(hri, tableName, edit, ee.currentTimeMillis()).checkedGet();
+ wal.append(hri, tableName, edit, ee.currentTimeMillis());
}
}
[37/50] [abbrv] git commit: [HBASE-11454] HBCK shouldn't check meta
and root region info against meta
Posted by ec...@apache.org.
[HBASE-11454] HBCK shouldn't check meta and root region info against meta
Summary: Meta doesn't contain region info for META and ROOT so the on disk region info can't be checked agains the meta one.
Test Plan: beholdunittests
Reviewers: pwhite, manukranthk, yuq
Reviewed By: yuq
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1415290
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43275 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/797de59a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/797de59a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/797de59a
Branch: refs/heads/0.89-fb
Commit: 797de59a219c76475867e42ffea4f324e0f15c08
Parents: 153336f
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jul 2 06:32:07 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/client/HBaseFsck.java | 3 +++
.../org/apache/hadoop/hbase/client/TestHBaseFsck.java | 12 ++++++++++++
2 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/797de59a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
index 6228151..659c8a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
@@ -267,6 +267,9 @@ public class HBaseFsck {
try {
for (HbckInfo hbi : regionInfo.values()) {
+ if (hbi.metaEntry.getTableDesc().isMetaTable()) {
+ continue;
+ }
tableDir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
hbi.metaEntry.getTableDesc().getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/797de59a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
index 16957d9..c34bc2b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
@@ -74,6 +74,18 @@ public class TestHBaseFsck {
@Test
+ public void testCheckRegionInfo() throws Exception {
+ HTableDescriptor desc = new HTableDescriptor("testCheckRegionInfo");
+ desc.addFamily(new HColumnDescriptor(Bytes.toBytes("D")));
+ admin.createTable(desc);
+ HBaseFsck fsck = new HBaseFsck(TEST_UTIL.getConfiguration());
+ Map<HRegionInfo, Path> result = fsck.checkRegionInfo();
+
+ assertEquals(0, result.size());
+ }
+
+
+ @Test
public void testFindMultipleHoles() throws IOException, InterruptedException {
byte[] tableName = Bytes.toBytes("testCreateTableWithRegions");
[25/50] [abbrv] git commit: [master] Encode/decode value of region
nane with URL-safe base64
Posted by ec...@apache.org.
[master] Encode/decode value of region nane with URL-safe base64
Summary: Since there may be some abnormal characters in region name.
Test Plan: Local web server.
Reviewers: elliott, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1396428
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43033 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e5a0d82
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e5a0d82
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e5a0d82
Branch: refs/heads/0.89-fb
Commit: 2e5a0d827235204ea82aca25b00e4b002b464069
Parents: 13c94a3
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Sat Jun 21 00:59:30 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hbase-webapps/master/assignmentPlan.jsp | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e5a0d82/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
index 10851bf..cb6ddbc 100644
--- a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
+++ b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
@@ -14,6 +14,7 @@
import="org.apache.hadoop.hbase.master.AssignmentPlan"
import="org.apache.hadoop.hbase.master.RegionPlacement"
import="org.apache.hadoop.hbase.master.RegionAssignmentSnapshot"
+ import="org.apache.hadoop.hbase.util.Base64"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.hbase.util.JvmVersion"
@@ -24,14 +25,21 @@
RegionAssignmentSnapshot snapShot = rp.getRegionAssignmentSnapshot();
Map<String, HRegionInfo> regionNameToRegionInfoMap = snapShot.getRegionNameToRegionInfoMap();
- String regionName = request.getParameter("regionName");
+ String regionNameStr = null;
+ String regionName64 = request.getParameter("regionName64");
+ if (regionName64 != null && regionName64.length() > 0) {
+ byte[] regionName = Base64.decode(regionName64, Base64.URL_SAFE);
+ if (regionName != null) {
+ regionNameStr = Bytes.toStringBinary(regionName);
+ }
+ }
String primaryRS = null;
String secondaryRS = null;
String tertiaryRS = null;
String favoredNodes = "";
String error = "";
- if (regionName != null) {
- HRegionInfo region = regionNameToRegionInfoMap.get(regionName);
+ if (regionNameStr != null) {
+ HRegionInfo region = regionNameToRegionInfoMap.get(regionNameStr);
List<HServerAddress> favoredNodesList = null;
if (region == null) {
error = " because cannot find this region in META !";
@@ -90,13 +98,13 @@
if (error.length() != 0) {
%>
<h3>Failed to update the favored nodes: <font color="#FF0000"><%=favoredNodes%> </font>, </h3>
- <h3>for the region: <font color="#FF0000"> <%= StringEscapeUtils.escapeHtml(regionName) %> </font></h3>
+ <h3>for the region: <font color="#FF0000"> <%= StringEscapeUtils.escapeHtml(regionNameStr) %> </font></h3>
<h3><%=error%></h3>
<%
- } else if (regionName != null) {
+ } else if (regionNameStr != null) {
%>
<h3> Succeeded to update the favored nodes: <font color="#32CD32"><%=favoredNodes%> </font>,</h3>
- <h3> for the region: <font color="#32CD32"><%= StringEscapeUtils.escapeHtml(regionName) %> </font></h3>
+ <h3> for the region: <font color="#32CD32"><%= StringEscapeUtils.escapeHtml(regionNameStr) %> </font></h3>
<%
}
%>
@@ -113,7 +121,8 @@
for (Map.Entry<String, HRegionInfo> entry : regionNameToRegionInfoMap.entrySet()) {
HRegionInfo regionInfo = entry.getValue();
List<HServerAddress> favoredNodeList = assignmentMap.get(regionInfo);
- regionName = regionInfo.getRegionNameAsString();
+ byte[] regionName = regionInfo.getRegionName();
+ regionNameStr = regionInfo.getRegionNameAsString();
String position = "Not Assigned";
String tdClass = "bad";
@@ -135,8 +144,8 @@
}
%>
<tr><form method="post">
- <input type="hidden" name="regionName" value="<%= StringEscapeUtils.escapeHtml(regionName) %>">
- <td><%= StringEscapeUtils.escapeHtml(regionName) %></td>
+ <input type="hidden" name="regionName64" value="<%= Base64.encodeBytes(regionName, Base64.URL_SAFE) %>">
+ <td><%= StringEscapeUtils.escapeHtml(regionNameStr) %></td>
<td class="<%= tdClass %>"><%= position %></td>
<td><input type="text" size="40" name="primaryRS" value="<%= favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.PRIMARY.ordinal()).getHostNameWithPort() %>"</td>
<td><input type="text" size="40" name="secondaryRS" value="<%= favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.SECONDARY.ordinal()).getHostNameWithPort() %>"</td>
[46/50] [abbrv] git commit: [master] Add test for
SizeAwareThriftHRegionServer
Posted by ec...@apache.org.
[master] Add test for SizeAwareThriftHRegionServer
Summary: Add a test
Test Plan: Unit test only
Reviewers: manukranthk, rshroff
Reviewed By: manukranthk, rshroff
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1409864
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43220 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4282d5d0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4282d5d0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4282d5d0
Branch: refs/heads/0.89-fb
Commit: 4282d5d0f2b6e5716a74eef7ae0953783c68889b
Parents: 6b30de6
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Sat Jun 28 01:34:14 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../SizeAwareThriftHRegionServer.java | 13 ++-
.../TestSizeAwareThriftHRegionServer.java | 116 +++++++++++++++++++
2 files changed, 123 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4282d5d0/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
index d2526c6..a0c4476 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
@@ -10,6 +10,7 @@ import org.apache.hadoop.hbase.client.MultiPut;
import org.apache.hadoop.hbase.client.MultiPutResponse;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.TMultiResponse;
+import org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException;
import javax.annotation.Nullable;
import java.util.List;
@@ -35,7 +36,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
@@ -51,7 +52,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
@@ -79,7 +80,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
@@ -97,7 +98,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
@@ -110,7 +111,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
@@ -126,7 +127,7 @@ public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
return f;
} else {
- return Futures.immediateFailedFuture(new RegionOverloadedException());
+ return Futures.immediateFailedFuture(new ThriftHBaseException(new RegionOverloadedException()));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4282d5d0/src/test/java/org/apache/hadoop/hbase/regionserver/TestSizeAwareThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSizeAwareThriftHRegionServer.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSizeAwareThriftHRegionServer.java
new file mode 100644
index 0000000..4b992dc
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSizeAwareThriftHRegionServer.java
@@ -0,0 +1,116 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertTrue;
+
+@Category(MediumTests.class)
+public class TestSizeAwareThriftHRegionServer {
+
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private ExecutorService service = Executors.newFixedThreadPool(20);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration configuration = TEST_UTIL.getConfiguration();
+ configuration.setLong("max.callqueue.memory.size", 1024 * 5);
+ configuration.setInt("hbase.client.retries.number", 3);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testSize() throws Exception {
+ String tableName = "testSizeThrottle";
+ final byte[] tableNameBytes = Bytes.toBytes(tableName);
+ final byte[] cf = Bytes.toBytes("d");
+ TEST_UTIL.createTable(tableNameBytes, cf);
+
+ HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+ ht.setAutoFlush(false);
+ Put smallPut = new Put(Bytes.toBytes("a"));
+ smallPut.add(cf, Bytes.toBytes(0), Bytes.toBytes("1"));
+
+ // This should work.
+ ht.put(smallPut);
+ ht.flushCommits();
+
+ final AtomicBoolean threw = new AtomicBoolean(false);
+
+ final ArrayList<Future<Void>> futures = new ArrayList<>();
+
+ // Since there's so many threads some of these should fail.
+ for (int i = 0; i < 20; i++) {
+ futures.add(service.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+
+ HTable t = new HTable(TEST_UTIL.getConfiguration(), tableNameBytes);
+ t.setAutoFlush(false);
+ for (int y = 0; y < 100; y++) {
+ Put p = new Put(Bytes.toBytes("test"));
+ for (int x = 0; x < 1000; x++) {
+ p.add(cf, Bytes.toBytes(x), Bytes.toBytes("b"));
+ }
+ try {
+ t.put(p);
+ t.flushCommits();
+ } catch (Exception roe) {
+ threw.set(true);
+ }
+
+ }
+
+ t.close();
+
+ return null;
+ }
+ }));
+ }
+
+ for (Future<Void> f : futures) {
+ f.get();
+ }
+
+ assertTrue("Should have caught a region too busy exception", threw.get());
+ }
+}
[14/50] [abbrv] git commit: [HBASE-11336] Show region sizes in table
page of master info server.
Posted by ec...@apache.org.
[HBASE-11336] Show region sizes in table page of master info server.
Summary: Extract sizes of regions on HDFS, and show them on info server table page.
Test Plan:
`TestFSUtils`
The web page on testcase.
Reviewers: manukranthk, adela, gauravm, elliott, fan
Reviewed By: fan
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1377657
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42820 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7403a0bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7403a0bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7403a0bf
Branch: refs/heads/0.89-fb
Commit: 7403a0bf209345e1cf3a2f666f39b1727c1bafe2
Parents: eae4c26
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 12 01:34:51 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +
.../org/apache/hadoop/hbase/util/FSUtils.java | 37 +++++++
.../resources/hbase-webapps/master/table.jsp | 28 ++++--
.../apache/hadoop/hbase/util/TestFSUtils.java | 100 +++++++++++++++++++
4 files changed, 159 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 07f559e..0d008ff 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1255,6 +1255,8 @@ public class HMaster extends HasThread implements HMasterInterface,
this.infoServer = new InfoServer(MASTER, a, port, false, conf);
this.infoServer.setAttribute(MASTER, this);
this.infoServer.start();
+ LOG.info("Master info server started at: " + a + ":"
+ + this.infoServer.getPort());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Started service threads");
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 7589ecf..6bfb630 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -27,6 +27,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
@@ -480,6 +481,42 @@ public class FSUtils {
}
/**
+ * Returns the sizes of some regions on DFS.
+ */
+ public static Map<HRegionInfo, Long> getRegionSizesInBytes(HMaster master,
+ Set<HRegionInfo> regions) throws IOException {
+ Configuration conf = master.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ return getRegionSizesInBytes(regions, fs, master.getRootDir());
+ }
+
+ /**
+ * Returns the sizes of some regions on DFS.
+ */
+ public static Map<HRegionInfo, Long> getRegionSizesInBytes(
+ Set<HRegionInfo> regions, FileSystem fs, Path hbaseRootDir)
+ throws IOException {
+ Map<HRegionInfo, Long> sizes = new ConcurrentHashMap<>();
+
+ DirFilter df = new DirFilter(fs);
+ for (HRegionInfo region : regions) {
+ long size = 0L;
+ Path regionDir = HRegion.getRegionDir(hbaseRootDir, region);
+ for (FileStatus familyDir : fs.listStatus(regionDir, df)) {
+ for (FileStatus storeFile : fs.listStatus(familyDir.getPath())) {
+ if (storeFile.isDir()) {
+ continue;
+ }
+ size += storeFile.getLen();
+ }
+ }
+ sizes.put(region, size);
+ }
+
+ return sizes;
+ }
+
+ /**
* Expects to find -ROOT- directory.
* @param fs filesystem
* @param hbaseRootDir hbase root directory
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index e690ae7..65bfa49 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -11,6 +11,8 @@
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.master.MetaRegion"
import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.util.StringUtils"
import="java.util.Map"
import="org.apache.hadoop.hbase.HConstants"%><%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
@@ -135,20 +137,30 @@
<%
Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
if(regions != null && regions.size() > 0) { %>
-<%= tableHeader %>
+ <h2>Table Regions</h2>
+ <table>
+ <tr>
+ <th>Name</th>
+ <th>Region Server</th>
+ <th>Start Key</th>
+ <th>End Key</th>
+ <th>Bytes</th>
+ </tr>
<%
+ Map<HRegionInfo, Long> regionSizes = FSUtils.getRegionSizesInBytes(master, regions.keySet());
for(Map.Entry<HRegionInfo, HServerAddress> hriEntry : regions.entrySet()) {
String urlRegionServer =
"http://" + hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort + "/";
%>
-<tr>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getRegionName())%></td>
- <td><a href="<%= urlRegionServer %>"><%= hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort %></a></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getStartKey())%></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getEndKey())%></td>
-</tr>
+ <tr>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getRegionName())%></td>
+ <td><a href="<%= urlRegionServer %>"><%= hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort %></a></td>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getStartKey())%></td>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getEndKey())%></td>
+ <td><%= StringUtils.byteDesc(regionSizes.get(hriEntry.getKey())) %></td>
+ </tr>
<% } %>
-</table>
+ </table>
<% }
} catch(Exception ex) {
ex.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
new file mode 100644
index 0000000..e118829
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -0,0 +1,100 @@
+/**
+ * Copyright 2014 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestFSUtils {
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ // Setting this make master start the info server.
+ conf.setInt(HConstants.MASTER_INFO_PORT, 0);
+ TEST_UTIL.startMiniCluster();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Test method for {@link org.apache.hadoop.hbase.util.FSUtils#getRegionSizesInBytes(java.util.Set, org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path)}.
+ */
+ @Test
+ public void testGetRegionSizesInBytes() throws Exception {
+ final StringBytes TABLE = new StringBytes("testGetRegionSizesInBytes");
+ final byte[] FAMILY = Bytes.toBytes("f");
+ Configuration conf = TEST_UTIL.getConfiguration();
+ Path rootDir = FSUtils.getRootDir(conf);
+ try (HTable table = TEST_UTIL.createTable(TABLE, FAMILY)) {
+ TEST_UTIL.loadTable(table, FAMILY);
+
+ // Flush to disk
+ for (HRegionServer rs : TEST_UTIL.getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ rs.flushRegion(region.getRegionName());
+ }
+ }
+
+ Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+ FileSystem fs = FileSystem.get(conf);
+ Map<HRegionInfo, Long> regionSizes =
+ FSUtils.getRegionSizesInBytes(regions.keySet(), fs, rootDir);
+ System.out.println(regionSizes);
+
+ Assert.assertEquals("Number of regions", regions.size(),
+ regionSizes.size());
+
+ for (HRegionInfo region : regions.keySet()) {
+ Assert.assertNotEquals(
+ "Size of region " + region.getRegionNameAsString(), 0L,
+ regionSizes.get(region).longValue());
+ }
+ }
+ }
+}
[31/50] [abbrv] git commit: [HBASE-11379] Modified AlterTable to fix
stale .regioninfo and wrote tests
Posted by ec...@apache.org.
[HBASE-11379] Modified AlterTable to fix stale .regioninfo and wrote tests
Summary:
changed the alterTable() method in HMaster to rewrite .regioninfo files (with parallel threads) after modification
incorporated David's comments
Test Plan: created Tables, modified them, and then verified that the .regioninfo files were still correct
Reviewers: adela, daviddeng, elliott, manukranthk
Reviewed By: elliott, manukranthk
Subscribers: daviddeng, elliott, adela, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1395379
Tasks: 4490604
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43138 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75ed75d3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75ed75d3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75ed75d3
Branch: refs/heads/0.89-fb
Commit: 75ed75d3ed5746f35817863d0ead400231bfd9ac
Parents: a45454d
Author: yuq <yu...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 22:35:16 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 56 +++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 192 +++++-----
.../apache/hadoop/hbase/client/HBaseFsck.java | 21 +-
.../hadoop/hbase/ipc/HMasterInterface.java | 11 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 380 +++++++++++--------
.../hadoop/hbase/regionserver/HRegion.java | 16 +-
.../client/TestHBaseFsckFixRegionInfo.java | 102 +++--
.../hbase/master/TestAlterTableRegionInfo.java | 152 ++++++++
8 files changed, 584 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 9b30994..63701cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -26,8 +26,14 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JenkinsHash;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.io.VersionedWritable;
@@ -115,6 +121,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return encodedName;
}
+ public static final char NEWLINE = '\n';
/** delimiter used between portions of a region name */
public static final int DELIMITER = ',';
@@ -696,4 +703,53 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
}
+
+ /**
+ * This method writes the .regioninfo file. This method is synchronized so that no two threads
+ * attempt to write the same .regioninfo file at the same time.
+ * @param conf
+ * @return true if the write was successful
+ * @throws IOException
+ */
+ public synchronized boolean writeToDisk(Configuration conf) throws IOException {
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ return writeToDisk(conf, fs);
+ }
+
+ /**
+ * This method writes the .regioninfo file. This method is synchronized so that no two threads
+ * attempt to write the same .regioninfo file at the same time.
+ * @param conf
+ * @param fs
+ * @return true if the write is successful
+ * @throws IOException
+ */
+ public synchronized boolean writeToDisk(Configuration conf, FileSystem fs) throws IOException {
+ Path tableDir =
+ HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), this.getTableDesc().getName());
+ Path regionPath = HRegion.getRegionDir(tableDir, this.getEncodedName());
+ Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
+ FSDataOutputStream out = fs.create(regionInfoPath, true);
+ boolean successfulWrite = true;
+ try {
+ // we write to file the information necessary to reconstruct this HRegionInfo via readFields()
+ // this.toString() calls tableDesc.toString(), so this file will contain back-up information
+ // for the table
+ write(out);
+ out.write(NEWLINE);
+ out.write(NEWLINE);
+ out.write(Bytes.toBytes(this.toString()));
+ LOG.debug("Rewrote .regioninfo file of " + this.getRegionNameAsString() + " at "
+ + regionInfoPath);
+ } catch (IOException e) {
+ successfulWrite = false;
+ LOG.error("Could not rewrite the .regioninfo of " + this.getRegionNameAsString() + " at "
+ + regionInfoPath, e);
+ } finally {
+ out.close();
+ LOG.debug(".regioninfo File Contents: " + this.toString());
+ }
+ return successfulWrite;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index c16889b..f090cbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -140,7 +140,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if the master is not running
*/
public boolean tableExists(final String tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
return tableExists(new StringBytes(tableName));
}
@@ -150,7 +150,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if the master is not running
*/
public boolean tableExists(final byte [] tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
return tableExists(new StringBytes(tableName));
}
@@ -183,7 +183,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public HTableDescriptor getTableDescriptor(final byte [] tableName)
- throws IOException {
+ throws IOException {
return this.connection.getHTableDescriptor(new StringBytes(tableName));
}
@@ -208,7 +208,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void createTable(HTableDescriptor desc)
- throws IOException {
+ throws IOException {
createTable(desc, null);
}
@@ -238,7 +238,7 @@ public class HBaseAdmin {
*/
public void createTable(HTableDescriptor desc, byte [] startKey,
byte [] endKey, int numRegions)
- throws IOException {
+ throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
if(numRegions < 3) {
throw new IllegalArgumentException("Must create at least three regions");
@@ -270,7 +270,7 @@ public class HBaseAdmin {
* @throws IOException
*/
public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
checkSplitKeys(splitKeys);
createTableAsync(desc, splitKeys);
@@ -347,8 +347,8 @@ public class HBaseAdmin {
for (byte[] splitKey : splitKeys) {
if (Bytes.equals(splitKey, HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException(
- "Split keys cannot be empty"
- );
+ "Split keys cannot be empty"
+ );
}
if (lastKey != null && Bytes.equals(splitKey, lastKey)) {
throw new IllegalArgumentException(
@@ -373,7 +373,7 @@ public class HBaseAdmin {
* @throws IOException
*/
public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
@@ -420,15 +420,15 @@ public class HBaseAdmin {
final int batchCount = this.conf.getInt("hbase.admin.scanner.caching", 10);
// Wait until first region is deleted
HRegionInterface server =
- connection.getHRegionConnection(firstMetaServer.getServerAddress());
+ connection.getHRegionConnection(firstMetaServer.getServerAddress());
HRegionInfo info = new HRegionInfo();
for (int tries = 0; tries < numRetries; tries++) {
long scannerId = -1L;
try {
Scan scan = new Scan().addColumn(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
+ HConstants.REGIONINFO_QUALIFIER);
scannerId = server.openScanner(
- firstMetaServer.getRegionInfo().getRegionName(), scan);
+ firstMetaServer.getRegionInfo().getRegionName(), scan);
// Get a batch at a time.
Result[] values = server.next(scannerId, batchCount);
if (values == null || values.length == 0) {
@@ -519,7 +519,7 @@ public class HBaseAdmin {
long sleep = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
- "enabled in " + Bytes.toStringBinary(tableName));
+ "enabled in " + Bytes.toStringBinary(tableName));
}
try {
Thread.sleep(sleep);
@@ -528,12 +528,12 @@ public class HBaseAdmin {
}
if (LOG.isDebugEnabled()) {
LOG.debug("Wake. Waiting for all regions to be enabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
}
if (!enabled)
throw new IOException("Unable to enable table " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
LOG.info("Enabled table " + Bytes.toStringBinary(tableName));
}
@@ -574,7 +574,7 @@ public class HBaseAdmin {
if (disabled) break;
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be disabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
try {
Thread.sleep(getPauseTime(tries));
@@ -583,12 +583,12 @@ public class HBaseAdmin {
}
if (LOG.isDebugEnabled()) {
LOG.debug("Wake. Waiting for all regions to be disabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
}
if (!disabled) {
throw new RegionException("Retries exhausted, it took too long to wait"+
- " for the table " + Bytes.toStringBinary(tableName) + " to be disabled.");
+ " for the table " + Bytes.toStringBinary(tableName) + " to be disabled.");
}
LOG.info("Disabled " + Bytes.toStringBinary(tableName));
}
@@ -649,15 +649,18 @@ public class HBaseAdmin {
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite the .regioninfo files that belong to the modified table
* Asynchronous operation.
*
* @param tableName name of the table to add column to
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files that belong to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final String tableName,
+ public boolean alterTable(final String tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<String, HColumnDescriptor>> columnModifications,
List<String> columnDeletions) throws IOException {
@@ -669,7 +672,7 @@ public class HBaseAdmin {
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications,
+ return alterTable(tableName, columnAdditions, columnModifications,
columnDeletions, waitInterval, maxClosedRegions);
}
@@ -686,14 +689,16 @@ public class HBaseAdmin {
* numConcurrentRegionsClosed over
* @param maxConcurrentRegionsClosed the max number of regions to have closed
* at a time.
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final String tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<String, HColumnDescriptor>> columnModifications,
- List<String> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
+ public boolean alterTable(final String tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<String, HColumnDescriptor>> columnModifications,
+ List<String> columnDeletions,
+ int waitInterval,
+ int maxConcurrentRegionsClosed) throws IOException {
// convert all of the strings to bytes and pass to the bytes method
List<Pair<byte [], HColumnDescriptor>> modificationsBytes =
new ArrayList<Pair<byte [], HColumnDescriptor>>(
@@ -701,21 +706,22 @@ public class HBaseAdmin {
List<byte []> deletionsBytes =
new ArrayList<byte []>(columnDeletions.size());
- for(Pair<String, HColumnDescriptor> c : columnModifications) {
- modificationsBytes.add(new Pair<byte [], HColumnDescriptor>(
- Bytes.toBytes(c.getFirst()), c.getSecond()));
- }
- for(String c : columnDeletions) {
- deletionsBytes.add(Bytes.toBytes(c));
- }
+ for(Pair<String, HColumnDescriptor> c : columnModifications) {
+ modificationsBytes.add(new Pair<byte [], HColumnDescriptor>(
+ Bytes.toBytes(c.getFirst()), c.getSecond()));
+ }
+ for(String c : columnDeletions) {
+ deletionsBytes.add(Bytes.toBytes(c));
+ }
- alterTable(Bytes.toBytes(tableName), columnAdditions, modificationsBytes,
- deletionsBytes, waitInterval, maxConcurrentRegionsClosed);
+ return alterTable(Bytes.toBytes(tableName), columnAdditions, modificationsBytes,
+ deletionsBytes, waitInterval, maxConcurrentRegionsClosed);
}
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite the .regioninfo files that belong to the modified table
* Any of the three lists can be null, in which case those types of
* alterations will be ignored.
* Asynchronous operation.
@@ -724,9 +730,11 @@ public class HBaseAdmin {
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte[], HColumnDescriptor>> columnModifications,
List<byte[]> columnDeletions) throws IOException {
@@ -739,12 +747,13 @@ public class HBaseAdmin {
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ return alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
}
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite all the .regioninfo files that belong to the modified table
* Any of the three lists can be null, in which case those types of
* alterations will be ignored.
* Asynchronous operation.
@@ -757,25 +766,27 @@ public class HBaseAdmin {
* numConcurrentRegionsClosed over
* @param maxConcurrentRegionsClosed the max number of regions to have closed
* at a time.
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte[], HColumnDescriptor>> columnModifications,
- List<byte[]> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
+ public boolean alterTable(final byte [] tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte[], HColumnDescriptor>> columnModifications,
+ List<byte[]> columnDeletions,
+ int waitInterval,
+ int maxConcurrentRegionsClosed) throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
HTableDescriptor.isLegalTableName(tableName);
try {
- this.master.alterTable(tableName, columnAdditions, columnModifications,
+ return this.master.alterTable(tableName, columnAdditions, columnModifications,
columnDeletions, waitInterval, maxConcurrentRegionsClosed);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
- }
+}
/**
* Get the status of alter command - indicates how many regions have received
@@ -808,7 +819,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void addColumn(final String tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), Arrays.asList(column), null, null);
}
@@ -821,7 +832,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void addColumn(final byte [] tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(tableName, Arrays.asList(column), null, null);
}
@@ -834,7 +845,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void deleteColumn(final String tableName, final String columnName)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), null, null,
Arrays.asList(Bytes.toBytes(columnName)));
}
@@ -849,7 +860,7 @@ public class HBaseAdmin {
*/
public void deleteColumn(final byte [] tableName,
final byte [] columnName)
- throws IOException {
+ throws IOException {
alterTable(tableName, null, null, Arrays.asList(columnName));
}
@@ -863,15 +874,15 @@ public class HBaseAdmin {
*/
public void modifyColumn(final String tableName, final String columnName,
HColumnDescriptor descriptor)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), null, Arrays.asList(
- new Pair<byte [], HColumnDescriptor>(Bytes.toBytes(columnName),
+ new Pair<byte [], HColumnDescriptor>(Bytes.toBytes(columnName),
descriptor)), null);
}
public void modifyColumn(final String tableName,
HColumnDescriptor descriptor)
- throws IOException {
+ throws IOException {
modifyColumn(tableName, descriptor.getNameAsString(), descriptor);
}
@@ -885,10 +896,10 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void modifyColumn(final byte [] tableName, final byte [] columnName,
- HColumnDescriptor descriptor)
- throws IOException {
+ HColumnDescriptor descriptor)
+ throws IOException {
alterTable(tableName, null, Arrays.asList(
- new Pair<byte [], HColumnDescriptor>(columnName, descriptor)), null);
+ new Pair<byte [], HColumnDescriptor>(columnName, descriptor)), null);
}
/**
@@ -901,7 +912,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void closeRegion(final String regionname, final Object... args)
- throws IOException {
+ throws IOException {
closeRegion(Bytes.toBytes(regionname), args);
}
@@ -915,7 +926,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void closeRegion(final byte [] regionname, final Object... args)
- throws IOException {
+ throws IOException {
// Be careful. Must match the handler over in HMaster at MODIFY_CLOSE_REGION
int len = (args == null)? 0: args.length;
int xtraArgsCount = 1;
@@ -925,7 +936,7 @@ public class HBaseAdmin {
System.arraycopy(args, 0, newargs, xtraArgsCount, len);
}
modifyTable(HConstants.META_TABLE_NAME, HConstants.Modify.CLOSE_REGION,
- newargs);
+ newargs);
}
/**
@@ -939,7 +950,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void moveRegion(final String regionName, final String regionServer)
- throws IOException {
+ throws IOException {
moveRegion(Bytes.toBytes(regionName), regionServer);
}
@@ -954,9 +965,9 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void moveRegion(final byte[] regionName, final String regionServer)
- throws IOException {
+ throws IOException {
modifyTable(HConstants.META_TABLE_NAME, HConstants.Modify.MOVE_REGION,
- new Object[]{regionName, regionServer});
+ new Object[]{regionName, regionServer});
}
/**
@@ -1012,7 +1023,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
private void compactCF(String tableName, String columnFamily, HConstants.Modify op)
- throws IOException {
+ throws IOException {
compactCF(Bytes.toBytes(tableName), Bytes.toBytes(columnFamily), op);
}
@@ -1025,11 +1036,11 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
private void compactCF(final byte[] tableName, final byte[] columnFamily, HConstants.Modify op)
- throws IOException {
+ throws IOException {
// Validate table name and column family.
if (!this.connection.tableExists(new StringBytes(tableName))) {
throw new IllegalArgumentException("HTable " + new StringBytes(tableName)
- + " does not exist");
+ + " does not exist");
} else if (!getTableDescriptor(tableName).hasFamily(columnFamily)) {
throw new IllegalArgumentException("Column Family "
+ new String(columnFamily) + " does not exist in "
@@ -1040,6 +1051,7 @@ public class HBaseAdmin {
HTable table = new HTable(this.conf, tableName);
Set <HRegionInfo> regions = table.getRegionsInfo().keySet();
Iterator <HRegionInfo> regionsIt = regions.iterator();
+ table.close();
// Iterate over all regions and send a compaction request to each.
while (regionsIt.hasNext()) {
@@ -1057,7 +1069,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void compact(String tableOrRegionName, String columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily, HConstants.Modify.TABLE_COMPACT);
return;
@@ -1080,7 +1092,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void compact(final byte[] tableOrRegionName, final byte[] columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily, HConstants.Modify.TABLE_COMPACT);
return;
@@ -1100,7 +1112,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final String tableNameOrRegionName)
- throws IOException {
+ throws IOException {
majorCompact(Bytes.toBytes(tableNameOrRegionName));
}
@@ -1112,7 +1124,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final byte [] tableNameOrRegionName)
- throws IOException {
+ throws IOException {
modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_MAJOR_COMPACT);
}
@@ -1125,7 +1137,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(String tableOrRegionName, String columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily,
HConstants.Modify.TABLE_MAJOR_COMPACT);
@@ -1148,7 +1160,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final byte[] tableOrRegionName, final byte[] columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily,
HConstants.Modify.TABLE_MAJOR_COMPACT);
@@ -1181,7 +1193,7 @@ public class HBaseAdmin {
}
public void split(final String tableNameOrRegionName,
- final String splitPoint) throws IOException {
+ final String splitPoint) throws IOException {
split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
}
@@ -1199,11 +1211,11 @@ public class HBaseAdmin {
throw new IllegalArgumentException("Pass a table name or region name");
}
byte [] tableName = tableExists(tableNameOrRegionName)?
- tableNameOrRegionName: null;
+ tableNameOrRegionName: null;
byte [] regionName = tableName == null? tableNameOrRegionName: null;
Object [] args = regionName == null?
- new byte [][] {splitPoint}: new byte [][] {regionName, splitPoint};
- modifyTable(tableName, HConstants.Modify.TABLE_EXPLICIT_SPLIT, args);
+ new byte [][] {splitPoint}: new byte [][] {regionName, splitPoint};
+ modifyTable(tableName, HConstants.Modify.TABLE_EXPLICIT_SPLIT, args);
}
/*
@@ -1215,7 +1227,7 @@ public class HBaseAdmin {
*/
private void modifyTable(final byte [] tableNameOrRegionName,
final HConstants.Modify op)
- throws IOException {
+ throws IOException {
if (tableNameOrRegionName == null) {
throw new IllegalArgumentException("Pass a table name or region name");
}
@@ -1236,7 +1248,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void modifyTable(final byte [] tableName, HTableDescriptor htd)
- throws IOException {
+ throws IOException {
modifyTable(tableName, HConstants.Modify.TABLE_SET_HTD, htd);
}
@@ -1252,7 +1264,7 @@ public class HBaseAdmin {
*/
public void modifyTable(final byte [] tableName, HConstants.Modify op,
Object... args)
- throws IOException {
+ throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
@@ -1266,7 +1278,7 @@ public class HBaseAdmin {
switch (op) {
case TABLE_SET_HTD:
if (args == null || args.length < 1 ||
- !(args[0] instanceof HTableDescriptor)) {
+ !(args[0] instanceof HTableDescriptor)) {
throw new IllegalArgumentException("SET_HTD requires a HTableDescriptor");
}
arr = new Writable[1];
@@ -1325,7 +1337,7 @@ public class HBaseAdmin {
return new BooleanWritable((Boolean) o);
} else {
throw new IllegalArgumentException("Requires byte [] or " +
- "ImmutableBytesWritable, not " + o.getClass() + " : " + o);
+ "ImmutableBytesWritable, not " + o.getClass() + " : " + o);
}
}
@@ -1358,8 +1370,8 @@ public class HBaseAdmin {
public synchronized void stopRegionServerForRestart(final HServerAddress hsa)
throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Restarting RegionServer" + hsa.toString());
- rs.stopForRestart();
+ LOG.info("Restarting RegionServer" + hsa.toString());
+ rs.stopForRestart();
}
/**
@@ -1393,9 +1405,9 @@ public class HBaseAdmin {
public synchronized void setNumHDFSQuorumReadThreads(final HServerAddress hsa,
int numThreads) throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Setting numHDFSQuorumReadThreads for RegionServer" + hsa.toString()
- + " to " + numThreads);
- rs.setNumHDFSQuorumReadThreads(numThreads);
+ LOG.info("Setting numHDFSQuorumReadThreads for RegionServer" + hsa.toString()
+ + " to " + numThreads);
+ rs.setNumHDFSQuorumReadThreads(numThreads);
}
/**
@@ -1409,9 +1421,9 @@ public class HBaseAdmin {
public synchronized void setHDFSQuorumReadTimeoutMillis(final HServerAddress hsa,
long timeoutMillis) throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Setting quorumReadTimeout for RegionServer" + hsa.toString()
- + " to " + timeoutMillis);
- rs.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
+ LOG.info("Setting quorumReadTimeout for RegionServer" + hsa.toString()
+ + " to " + timeoutMillis);
+ rs.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
}
/**
@@ -1427,9 +1439,9 @@ public class HBaseAdmin {
}
private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
- throws IOException {
+ throws IOException {
return connection.locateRegion(HConstants.META_TABLE_NAME_STRINGBYTES,
- HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
+ HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
}
/**
@@ -1439,7 +1451,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if a remote or network exception occurs
*/
public static void checkHBaseAvailable(Configuration conf)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
copyOfConf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 1);
new HBaseAdmin(copyOfConf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
index 5206f4a..6228151 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
@@ -46,7 +46,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -328,27 +327,9 @@ public class HBaseFsck {
void fixRegionInfo() throws IOException {
Map<HRegionInfo, Path> risToRewrite = checkRegionInfo();
LOG.info("FOUND " + risToRewrite.size() + " corrupted .regioninfo files to rewrite");
- Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
- FileSystem fs = rootDir.getFileSystem(conf);
for(Map.Entry<HRegionInfo, Path> entry : risToRewrite.entrySet()){
HRegionInfo hri = entry.getKey();
- Path regionInfoPath = entry.getValue();
- FSDataOutputStream out = fs.create(regionInfoPath, true);
-
- try {
- hri.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(hri.toString()));
- LOG.debug("Rewrote .regioninfo file of " + Bytes.toString(hri.getRegionName()) +"at " + regionInfoPath);
- } catch (IOException e){
- errors.reportError("Could not rewrite the .regioninfo of "+Bytes.toString(hri.getRegionName()) + " at " +regionInfoPath);
- }
-
- finally {
- out.close();
- LOG.debug(".regioninfo File Contents: " + Bytes.toBytes(hri.toString()));
- }
+ hri.writeToDisk(conf);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index c09f117..5a9bb5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.Writable;
@@ -69,14 +68,17 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion, ThriftClientI
* Batch adds, modifies, and deletes columns from the specified table.
* Any of the lists may be null, in which case those types of alterations
* will not occur.
+ * .regioninfo files in the relevant table will be properly rewritten after
+ * the modification
*
* @param tableName table to modify
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files in the table were properly rewritten
* @throws IOException e
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte [], HColumnDescriptor>> columnModifications,
List<byte []> columnDeletions) throws IOException;
@@ -85,14 +87,17 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion, ThriftClientI
* Batch adds, modifies, and deletes columns from the specified table.
* Any of the lists may be null, in which case those types of alterations
* will not occur.
+ * .regioninfo files in the relevant table will be properly rewritten after
+ * the modification
*
* @param tableName table to modify
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files in the table were properly rewritten
* @throws IOException e
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte [], HColumnDescriptor>> columnModifications,
List<byte []> columnDeletions,
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c95f821..9b51280 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -40,7 +40,11 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Set;
import java.util.SortedMap;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -142,7 +146,7 @@ import com.google.common.collect.Lists;
* @see Watcher
*/
public class HMaster extends HasThread implements HMasterInterface,
- HMasterRegionInterface, Watcher, StoppableMaster {
+HMasterRegionInterface, Watcher, StoppableMaster {
// MASTER is name of the webapp and the attribute name used stuffing this
//instance into web context.
public static final String MASTER = "master";
@@ -361,7 +365,7 @@ public class HMaster extends HasThread implements HMasterInterface,
LOG.debug("HMaster started in backup mode. Stall " + stallTime +
"ms giving primary master a fair chance to be the master...");
try {
- Thread.sleep(stallTime);
+ Thread.sleep(stallTime);
} catch (InterruptedException e) {
// interrupted = user wants to kill us. Don't continue
throw new IOException("Interrupted waiting for master address");
@@ -371,7 +375,7 @@ public class HMaster extends HasThread implements HMasterInterface,
final String masterName = getServerName();
// initialize the thread pool for non-distributed log splitting.
int maxSplitLogThread =
- conf.getInt("hbase.master.splitLogThread.max", 1000);
+ conf.getInt("hbase.master.splitLogThread.max", 1000);
logSplitThreadPool = Threads.getBoundedCachedThreadPool(
maxSplitLogThread, 30L, TimeUnit.SECONDS,
new ThreadFactory() {
@@ -391,18 +395,18 @@ public class HMaster extends HasThread implements HMasterInterface,
// Only read favored nodes if using the assignment-based load balancer.
this.shouldAssignRegionsWithFavoredNodes = conf.getClass(
HConstants.LOAD_BALANCER_IMPL, Object.class).equals(
- RegionManager.AssignmentLoadBalancer.class);
+ RegionManager.AssignmentLoadBalancer.class);
LOG.debug("Whether to read the favoredNodes from meta: " +
(shouldAssignRegionsWithFavoredNodes ? "Yes" : "No"));
// Initialize table level lock manager for schema changes, if enabled.
if (conf.getBoolean(HConstants.MASTER_SCHEMA_CHANGES_LOCK_ENABLE,
- HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_ENABLE)) {
+ HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_ENABLE)) {
int schemaChangeLockTimeoutMs = conf.getInt(
- HConstants.MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS,
- HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS);
+ HConstants.MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS,
+ HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS);
tableLockManager = new TableLockManager(zooKeeperWrapper,
- address, schemaChangeLockTimeoutMs);
+ address, schemaChangeLockTimeoutMs);
this.schemaChangeTryLockTimeoutMs = conf.getInt(
HConstants.MASTER_SCHEMA_CHANGES_TRY_LOCK_TIMEOUT_MS,
@@ -477,8 +481,8 @@ public class HMaster extends HasThread implements HMasterInterface,
this.getConfigurationManager().registerObserver(serverManager);
this.regionServerOperationQueue =
- new RegionServerOperationQueue(this.conf, serverManager,
- getClosedStatus());
+ new RegionServerOperationQueue(this.conf, serverManager,
+ getClosedStatus());
// start the "close region" executor service
HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(
@@ -539,16 +543,16 @@ public class HMaster extends HasThread implements HMasterInterface,
zooKeeperWrapper = ZooKeeperWrapper.createInstance(localConf,
getZKWrapperName(), new Abortable() {
- @Override
- public void abort(String why, Throwable e) {
- stop("ZK session expired");
- }
+ @Override
+ public void abort(String why, Throwable e) {
+ stop("ZK session expired");
+ }
- @Override
- public boolean isAborted() {
- return stopped;
- }
- });
+ @Override
+ public boolean isAborted() {
+ return stopped;
+ }
+ });
}
}
@@ -595,8 +599,8 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
private static Path checkRootDir(final Path rd, final Configuration c,
- final FileSystem fs)
- throws IOException {
+ final FileSystem fs)
+ throws IOException {
// If FS is in safe mode wait till out of it.
FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000));
@@ -615,7 +619,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
private static void bootstrap(final Path rd, final Configuration c)
- throws IOException {
+ throws IOException {
LOG.info("BOOTSTRAP: creating ROOT and first META regions");
try {
// Bootstrapping, make sure blockcache is off. Else, one will be
@@ -623,21 +627,21 @@ public class HMaster extends HasThread implements HMasterInterface,
// not make it in first place. Turn off block caching for bootstrap.
// Enable after.
HRegionInfo rootHRI = HTableDescriptor.isMetaregionSeqidRecordEnabled(c) ?
- new HRegionInfo(HRegionInfo.ROOT_REGIONINFO_WITH_HISTORIAN_COLUMN) :
- new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
- setInfoFamilyCaching(rootHRI, false);
- HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
- setInfoFamilyCaching(metaHRI, false);
- HRegion root = HRegion.createHRegion(rootHRI, rd, c);
- HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
- setInfoFamilyCaching(rootHRI, true);
- setInfoFamilyCaching(metaHRI, true);
- // Add first region from the META table to the ROOT region.
- HRegion.addRegionToMETA(root, meta);
- root.close();
- root.getLog().closeAndDelete();
- meta.close();
- meta.getLog().closeAndDelete();
+ new HRegionInfo(HRegionInfo.ROOT_REGIONINFO_WITH_HISTORIAN_COLUMN) :
+ new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
+ setInfoFamilyCaching(rootHRI, false);
+ HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+ setInfoFamilyCaching(metaHRI, false);
+ HRegion root = HRegion.createHRegion(rootHRI, rd, c);
+ HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
+ setInfoFamilyCaching(rootHRI, true);
+ setInfoFamilyCaching(metaHRI, true);
+ // Add first region from the META table to the ROOT region.
+ HRegion.addRegionToMETA(root, meta);
+ root.close();
+ root.getLog().closeAndDelete();
+ meta.close();
+ meta.getLog().closeAndDelete();
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.error("bootstrap", e);
@@ -663,10 +667,10 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws UnknownHostException
*/
private static String getMyAddress(final Configuration c)
- throws UnknownHostException, SocketException {
+ throws UnknownHostException, SocketException {
// Find out our address up in DNS.
String s = DNS.getDefaultHost(c.get("hbase.master.dns.interface","default"),
- c.get("hbase.master.dns.nameserver","default"));
+ c.get("hbase.master.dns.nameserver","default"));
if (preferIpv6AddressForMaster(c)) {
// Use IPv6 address if possible.
s = HServerInfo.getIPv6AddrIfLocalMachine(s);
@@ -818,7 +822,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
MonitoredTask startupStatus =
- TaskMonitor.get().createStatus("Master startup");
+ TaskMonitor.get().createStatus("Master startup");
startupStatus.setDescription("Master startup");
clusterStateRecovery = new ZKClusterStateRecovery(this, connection);
try {
@@ -887,19 +891,19 @@ public class HMaster extends HasThread implements HMasterInterface,
break;
} else {
LOG.debug("Waiting on " +
- this.serverManager.getServersToServerInfo().keySet().toString());
+ this.serverManager.getServersToServerInfo().keySet().toString());
}
}
switch (this.regionServerOperationQueue.process()) {
case FAILED:
- // If FAILED op processing, bad. Exit.
+ // If FAILED op processing, bad. Exit.
break FINISHED;
case REQUEUED_BUT_PROBLEM:
// LOG if the file system is down, but don't do anything.
checkFileSystem(false);
break;
default:
- // Continue run loop if conditions are PROCESSED, NOOP, REQUEUED
+ // Continue run loop if conditions are PROCESSED, NOOP, REQUEUED
break;
}
}
@@ -944,8 +948,8 @@ public class HMaster extends HasThread implements HMasterInterface,
private void initPreferredAssignment() {
// assign the regions based on the region locality in this period of time
this.applyPreferredAssignmentPeriod =
- conf.getLong("hbase.master.applyPreferredAssignment.period",
- 5 * 60 * 1000);
+ conf.getLong("hbase.master.applyPreferredAssignment.period",
+ 5 * 60 * 1000);
// disable scanning dfs by setting applyPreferredAssignmentPeriod to 0
if (applyPreferredAssignmentPeriod > 0) {
@@ -953,8 +957,8 @@ public class HMaster extends HasThread implements HMasterInterface,
// since master startup, then the master is free to assign this region
// out to any region server
this.holdRegionForBestLocalityPeriod =
- conf.getLong("hbase.master.holdRegionForBestLocality.period",
- 1 * 60 * 1000);
+ conf.getLong("hbase.master.holdRegionForBestLocality.period",
+ 1 * 60 * 1000);
// try to get the locality map from disk
this.preferredRegionToRegionServerMapping = getRegionLocalityFromSnapshot(conf);
@@ -969,24 +973,24 @@ public class HMaster extends HasThread implements HMasterInterface,
public static MapWritable getRegionLocalityFromSnapshot(Configuration conf) {
String region_assignment_snapshot_dir =
- conf.get("hbase.tmp.dir");
+ conf.get("hbase.tmp.dir");
if (region_assignment_snapshot_dir == null) {
return null;
}
String region_assignment_snapshot =
- region_assignment_snapshot_dir + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+ region_assignment_snapshot_dir + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
long refresh_interval =
- conf.getLong("hbase.master.regionLocality.snapshot.validity_time_ms",
- 24 * 60 * 60 * 1000);
+ conf.getLong("hbase.master.regionLocality.snapshot.validity_time_ms",
+ 24 * 60 * 60 * 1000);
File snapshotFile = new File(region_assignment_snapshot);
try {
if (!snapshotFile.exists()) {
- LOG.info("preferredRegionToRegionServerMapping snapshot not found. File Path: "
- + region_assignment_snapshot);
- return null;
+ LOG.info("preferredRegionToRegionServerMapping snapshot not found. File Path: "
+ + region_assignment_snapshot);
+ return null;
}
long time_elapsed = System.currentTimeMillis() - snapshotFile.lastModified();
@@ -1028,10 +1032,10 @@ public class HMaster extends HasThread implements HMasterInterface,
LOG.debug("Evaluate preferredRegionToRegionServerMapping; expecting pause here");
try {
regionLocalityMap = FSUtils
- .getRegionLocalityMappingFromFS(FileSystem.get(conf), FSUtils.getRootDir(conf),
- poolSize,
- conf,
- tablename);
+ .getRegionLocalityMappingFromFS(FileSystem.get(conf), FSUtils.getRootDir(conf),
+ poolSize,
+ conf,
+ tablename);
} catch (Exception e) {
LOG.error("Got unexpected exception when evaluating " +
"preferredRegionToRegionServerMapping : " + e.toString());
@@ -1047,7 +1051,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
String region_assignment_snapshot = tmp_path
- + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+ + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
// write the preferredRegionAssignment to disk
try {
LOG.info("Saving preferredRegionToRegionServerMapping " +
@@ -1055,8 +1059,8 @@ public class HMaster extends HasThread implements HMasterInterface,
regionLocalityMap.write(new DataOutputStream(
new FileOutputStream(region_assignment_snapshot)));
} catch (IOException e) {
- LOG.error("Error saving preferredRegionToRegionServerMapping " +
- "to file " + region_assignment_snapshot + " : " + e.toString());
+ LOG.error("Error saving preferredRegionToRegionServerMapping " +
+ "to file " + region_assignment_snapshot + " : " + e.toString());
}
return regionLocalityMap;
}
@@ -1298,18 +1302,18 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public MapWritable regionServerStartup(final HServerInfo serverInfo)
- throws IOException {
+ throws IOException {
// Set the ip into the passed in serverInfo. Its ip is more than likely
// not the ip that the master sees here. See at end of this method where
// we pass it back to the regionserver by setting "hbase.regionserver.address"
String rsAddress = HBaseServer.getRemoteAddress();
serverInfo.setServerAddress(new HServerAddress(rsAddress,
- serverInfo.getServerAddress().getPort()));
+ serverInfo.getServerAddress().getPort()));
// Register with server manager
this.serverManager.regionServerStartup(serverInfo);
// Send back some config info
MapWritable mw = createConfigurationSubset();
- mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
+ mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
return mw;
}
@@ -1329,10 +1333,10 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public HMsg [] regionServerReport(HServerInfo serverInfo, HMsg msgs[],
- HRegionInfo[] mostLoadedRegions)
- throws IOException {
+ HRegionInfo[] mostLoadedRegions)
+ throws IOException {
return adornRegionServerAnswer(serverInfo,
- this.serverManager.regionServerReport(serverInfo, msgs, mostLoadedRegions));
+ this.serverManager.regionServerReport(serverInfo, msgs, mostLoadedRegions));
}
void updateLastFlushedSequenceIds(HServerInfo serverInfo) {
@@ -1349,7 +1353,7 @@ public class HMaster extends HasThread implements HMasterInterface,
+ Bytes.toStringBinary(entry.getKey()) + " Ignoring.");
}
continue; // Don't let smaller sequence ids override greater
- // sequence ids.
+ // sequence ids.
}
}
flushedSequenceIdByRegion.put(entry.getKey(), entry.getValue());
@@ -1429,14 +1433,14 @@ public class HMaster extends HasThread implements HMasterInterface,
}
protected void lockTable(byte[] tableName, String purpose)
- throws IOException {
+ throws IOException {
if (isTableLockEnabled()) {
tableLockManager.lockTable(tableName, purpose);
}
}
protected void unlockTable(byte[] tableName)
- throws IOException {
+ throws IOException {
if (isTableLockEnabled()) {
tableLockManager.unlockTable(tableName);
}
@@ -1475,7 +1479,7 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
HRegionInfo[] newRegions = createRegionsForNewTable(desc, splitKeys);
try {
// We can not create a table unless meta regions have already been
@@ -1492,14 +1496,14 @@ public class HMaster extends HasThread implements HMasterInterface,
throw e;
} catch (IOException e) {
LOG.error("Cannot create table " + desc.getNameAsString() +
- " because of " + e.toString());
+ " because of " + e.toString());
throw RemoteExceptionHandler.checkIOException(e);
}
}
private static boolean tableExists(HRegionInterface srvr,
byte[] metaRegionName, String tableName)
- throws IOException {
+ throws IOException {
byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
Scan scan = new Scan(firstRowInTable);
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
@@ -1509,8 +1513,8 @@ public class HMaster extends HasThread implements HMasterInterface,
BaseScanner.getOneResultFromScanner(srvr, scannerid);
if (data != null && data.size() > 0) {
HRegionInfo info = Writables.getHRegionInfo(
- data.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER));
+ data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
if (info.getTableDesc().getNameAsString().equals(tableName)) {
// A region for this table already exists. Ergo table exists.
return true;
@@ -1569,7 +1573,7 @@ public class HMaster extends HasThread implements HMasterInterface,
if (assignmentPlan != null) {
// create the region with favorite nodes.
List<HServerAddress> favoredNodes =
- assignmentPlan.getAssignment(newRegion);
+ assignmentPlan.getAssignment(newRegion);
regionManager.createRegion(newRegion, srvr, metaRegionName,
favoredNodes);
} else {
@@ -1611,33 +1615,47 @@ public class HMaster extends HasThread implements HMasterInterface,
}
@Override
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte [], HColumnDescriptor>> columnModifications,
- List<byte []> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
- //This lock will be released when the ThrottledRegionReopener is done.
+ /**
+ * returns true if all .regioninfo files were succesfully rewritten after alteration
+ */
+ public boolean alterTable(final byte[] tableName, List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte[], HColumnDescriptor>> columnModifications, List<byte[]> columnDeletions,
+ int waitInterval, int maxConcurrentRegionsClosed) throws IOException {
+ // This lock will be released when the ThrottledRegionReopener is done.
if (!tryLockTable(tableName, "alter", schemaChangeTryLockTimeoutMs)) {
throw new TableLockTimeoutException("Timed out acquiring lock for "
- + Bytes.toStringBinary(tableName) + " after "
- + schemaChangeTryLockTimeoutMs + " ms.");
+ + Bytes.toStringBinary(tableName) + " after " + schemaChangeTryLockTimeoutMs + " ms.");
}
InjectionHandler.processEvent(InjectionEvent.HMASTER_ALTER_TABLE);
- ThrottledRegionReopener reopener = this.regionManager.
- createThrottledReopener(Bytes.toString(tableName), waitInterval, maxConcurrentRegionsClosed);
+ ThrottledRegionReopener reopener =
+ this.regionManager.createThrottledReopener(Bytes.toString(tableName), waitInterval,
+ maxConcurrentRegionsClosed);
// Regions are added to the reopener in MultiColumnOperation
- new MultiColumnOperation(this, tableName, columnAdditions,
- columnModifications, columnDeletions).process();
+ new MultiColumnOperation(this, tableName, columnAdditions, columnModifications, columnDeletions)
+ .process();
reopener.startRegionsReopening();
+
+ // after the alteration, all .regioninfo files for this table will be old so we rewrite them.
+ try {
+ writeRegionInfo(tableName);
+ } catch (Exception e) {
+ LOG.error(
+ "Failed to complete rewriting .regioninfo files for table "
+ + Bytes.toStringBinary(tableName), e);
+ return false;
+ }
+ return true;
}
@Override
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte [], HColumnDescriptor>> columnModifications,
- List<byte []> columnDeletions) throws IOException {
+ /**
+ * Returns true if all backup .regioninfo files were successfully rewritten
+ */
+ public boolean alterTable(final byte [] tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte [], HColumnDescriptor>> columnModifications,
+ List<byte []> columnDeletions) throws IOException {
int waitInterval = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_WAIT_INTERVAL_MS,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_WAIT_INTERVAL_MS);
@@ -1645,7 +1663,45 @@ public class HMaster extends HasThread implements HMasterInterface,
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ return alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ }
+
+ /**
+ * This method rewrites all .regioninfo files for all regions belonging to the Table whose name is
+ * tableName
+ * @param tableName
+ * @return true if all .regioninfo files that belong to the table are successfully rewritten.
+ * @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
+ public boolean writeRegionInfo(byte[] tableName) throws Exception {
+ List<Pair<HRegionInfo, HServerAddress>> tableRegions = getTableRegions(tableName);
+ ExecutorService executor = Executors.newCachedThreadPool();
+ List<Future<Boolean>> futures = new ArrayList<>();
+ for (final Pair<HRegionInfo, HServerAddress> entry : tableRegions) {
+ Callable<Boolean> writer = new Callable<Boolean>() {
+
+ // return true if the write is successful
+ public Boolean call() {
+ HRegionInfo hri = entry.getFirst();
+ try {
+ return hri.writeToDisk(conf);
+ } catch (IOException e) {
+ LOG.error("Failed to write .regioninfo for Region: " + hri.getRegionNameAsString(), e);
+ return false;
+ }
+ }
+ };
+ Future<Boolean> future = executor.submit(writer);
+ futures.add(future);
+ }
+ for (Future<Boolean> f : futures) {
+ if (!f.get()) { // if any write was not successful
+ return false;
+ }
+ }
+ return true;
}
@Override
@@ -1654,7 +1710,7 @@ public class HMaster extends HasThread implements HMasterInterface,
Pair <Integer, Integer> p = new Pair<Integer, Integer>(0,0);
if (regionManager.getThrottledReopener(Bytes.toString(tableName)) != null) {
p = regionManager.getThrottledReopener(
- Bytes.toString(tableName)).getReopenStatus();
+ Bytes.toString(tableName)).getReopenStatus();
} else {
// Table is not reopening any regions return (0,0)
}
@@ -1663,21 +1719,21 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public void addColumn(byte [] tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(tableName, Arrays.asList(column), null, null);
}
@Override
public void modifyColumn(byte [] tableName, byte [] columnName,
- HColumnDescriptor descriptor)
- throws IOException {
+ HColumnDescriptor descriptor)
+ throws IOException {
alterTable(tableName, null, Arrays.asList(
new Pair<byte[], HColumnDescriptor>(columnName, descriptor)), null);
}
@Override
public void deleteColumn(final byte [] tableName, final byte [] c)
- throws IOException {
+ throws IOException {
alterTable(tableName, null, null,
Arrays.asList(KeyValue.parseColumn(c)[0]));
}
@@ -1718,27 +1774,27 @@ public class HMaster extends HasThread implements HMasterInterface,
*/
public List<Pair<HRegionInfo,HServerAddress>> getTableRegions(
final byte [] tableName)
- throws IOException {
+ throws IOException {
final ArrayList<Pair<HRegionInfo, HServerAddress>> result =
- Lists.newArrayList();
+ Lists.newArrayList();
if (!Bytes.equals(HConstants.META_TABLE_NAME, tableName)) {
MetaScannerVisitor visitor =
- new MetaScannerVisitor() {
- @Override
- public boolean processRow(Result data) throws IOException {
- if (data == null || data.size() <= 0)
- return true;
- Pair<HRegionInfo, HServerAddress> pair =
- metaRowToRegionPair(data);
- if (pair == null) return false;
- if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
- tableName)) {
- return false;
- }
- result.add(pair);
+ new MetaScannerVisitor() {
+ @Override
+ public boolean processRow(Result data) throws IOException {
+ if (data == null || data.size() <= 0)
return true;
+ Pair<HRegionInfo, HServerAddress> pair =
+ metaRowToRegionPair(data);
+ if (pair == null) return false;
+ if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
+ tableName)) {
+ return false;
}
+ result.add(pair);
+ return true;
+ }
};
MetaScanner.metaScan(conf, visitor, new StringBytes(tableName));
@@ -1748,7 +1804,7 @@ public class HMaster extends HasThread implements HMasterInterface,
for (MetaRegion mRegion: metaRegions) {
if (Bytes.equals(mRegion.getRegionInfo().getTableDesc().getName(), tableName)) {
result.add(new Pair<HRegionInfo, HServerAddress>
- (mRegion.getRegionInfo(), mRegion.getServer()));
+ (mRegion.getRegionInfo(), mRegion.getServer()));
}
}
}
@@ -1779,26 +1835,26 @@ public class HMaster extends HasThread implements HMasterInterface,
*/
Pair<HRegionInfo,HServerAddress> getTableRegionForRow(
final byte [] tableName, final byte [] rowKey)
- throws IOException {
+ throws IOException {
final AtomicReference<Pair<HRegionInfo, HServerAddress>> result =
- new AtomicReference<Pair<HRegionInfo, HServerAddress>>(null);
+ new AtomicReference<Pair<HRegionInfo, HServerAddress>>(null);
MetaScannerVisitor visitor =
- new MetaScannerVisitor() {
- @Override
- public boolean processRow(Result data) throws IOException {
- if (data == null || data.size() <= 0)
- return true;
- Pair<HRegionInfo, HServerAddress> pair =
- metaRowToRegionPair(data);
- if (pair == null) return false;
- if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
- tableName)) {
- return false;
- }
- result.set(pair);
+ new MetaScannerVisitor() {
+ @Override
+ public boolean processRow(Result data) throws IOException {
+ if (data == null || data.size() <= 0)
return true;
+ Pair<HRegionInfo, HServerAddress> pair =
+ metaRowToRegionPair(data);
+ if (pair == null) return false;
+ if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
+ tableName)) {
+ return false;
}
+ result.set(pair);
+ return true;
+ }
};
MetaScanner.metaScan(conf, visitor, new StringBytes(tableName), rowKey, 1);
@@ -1808,7 +1864,7 @@ public class HMaster extends HasThread implements HMasterInterface,
@SuppressWarnings("deprecation")
Pair<HRegionInfo,HServerAddress> getTableRegionFromName(
final byte [] regionName)
- throws IOException {
+ throws IOException {
byte [] tableName = HRegionInfo.parseRegionName(regionName)[0];
Set<MetaRegion> regions = regionManager.getMetaRegionsForTable(tableName);
@@ -1834,7 +1890,7 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
protected Result getFromMETA(final byte[] row, final byte[] family)
- throws IOException {
+ throws IOException {
MetaRegion meta = this.regionManager.getMetaRegionForRow(row);
HRegionInterface srvr = getMETAServer(meta);
Get get = new Get.Builder(row).addFamily(family).create();
@@ -1847,7 +1903,7 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
private HRegionInterface getMETAServer(final MetaRegion meta)
- throws IOException {
+ throws IOException {
return this.connection.getHRegionConnection(meta.getServer());
}
@@ -1858,18 +1914,18 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException if a remote or network exception occurs
*/
public HTableDescriptor getTableDescriptor(final byte [] tableName)
- throws IOException {
+ throws IOException {
return this.connection.getHTableDescriptor(new StringBytes(tableName));
}
@Override
public void modifyTable(final byte[] tableName, HConstants.Modify op,
Writable[] args)
- throws IOException {
+ throws IOException {
switch (op) {
case TABLE_SET_HTD:
if (args == null || args.length < 1 ||
- !(args[0] instanceof HTableDescriptor))
+ !(args[0] instanceof HTableDescriptor))
throw new IOException("SET_HTD request requires an HTableDescriptor");
HTableDescriptor htd = (HTableDescriptor) args[0];
LOG.info("modifyTable(SET_HTD): " + htd);
@@ -1883,7 +1939,7 @@ public class HMaster extends HasThread implements HMasterInterface,
if (args != null && args.length > 0) {
if (!(args[0] instanceof ImmutableBytesWritable))
throw new IOException(
- "request argument must be ImmutableBytesWritable");
+ "request argument must be ImmutableBytesWritable");
Pair<HRegionInfo,HServerAddress> pair = null;
if(tableName == null) {
byte [] regionName = ((ImmutableBytesWritable)args[0]).get();
@@ -1892,8 +1948,8 @@ public class HMaster extends HasThread implements HMasterInterface,
byte [] rowKey = ((ImmutableBytesWritable)args[0]).get();
pair = getTableRegionForRow(tableName, rowKey);
}
- LOG.info("About to " + op.toString() + " on "
- + Bytes.toStringBinary(tableName) + " and pair is " + pair);
+ LOG.info("About to " + op.toString() + " on "
+ + Bytes.toStringBinary(tableName) + " and pair is " + pair);
if (pair != null && pair.getSecond() != null) {
// If the column family name is specified, we need to perform a
// column family specific action instead of an action on the whole
@@ -1916,12 +1972,12 @@ public class HMaster extends HasThread implements HMasterInterface,
for (Pair<HRegionInfo,HServerAddress> pair: getTableRegions(tableName)) {
if (pair.getSecond() == null) continue; // undeployed
this.regionManager.startAction(pair.getFirst().getRegionName(),
- pair.getFirst(), pair.getSecond(), op);
+ pair.getFirst(), pair.getSecond(), op);
}
}
break;
- // format : {tableName row | region} splitPoint
+ // format : {tableName row | region} splitPoint
case TABLE_EXPLICIT_SPLIT:
if (args == null || args.length < (tableName == null? 2 : 1)) {
throw new IOException("incorrect number of arguments given");
@@ -1947,13 +2003,13 @@ public class HMaster extends HasThread implements HMasterInterface,
}
HRegionInfo r = pair.getFirst();
r.setSplitPoint(splitPoint);
- LOG.info("About to " + op.toString() + " on "
- + Bytes.toStringBinary(pair.getFirst().getTableDesc().getName())
- + " at " + Bytes.toStringBinary(splitPoint) + " and pair is "
- + pair);
+ LOG.info("About to " + op.toString() + " on "
+ + Bytes.toStringBinary(pair.getFirst().getTableDesc().getName())
+ + " at " + Bytes.toStringBinary(splitPoint) + " and pair is "
+ + pair);
if (pair.getSecond() != null) {
this.regionManager.startAction(pair.getFirst().getRegionName(),
- pair.getFirst(), pair.getSecond(), Modify.TABLE_SPLIT);
+ pair.getFirst(), pair.getSecond(), Modify.TABLE_SPLIT);
}
break;
@@ -1976,7 +2032,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
this.regionManager.getAssignmentManager().
- addTransientAssignment(serverAddress, hri);
+ addTransientAssignment(serverAddress, hri);
// Close the region so that it will be re-opened by the preferred host.
modifyTable(tableName, HConstants.Modify.CLOSE_REGION, new Writable[]{args[0]});
@@ -1986,7 +2042,7 @@ public class HMaster extends HasThread implements HMasterInterface,
case CLOSE_REGION:
if (args == null || args.length < 1 || args.length > 2) {
throw new IOException("Requires at least a region name; " +
- "or cannot have more than region name and servername");
+ "or cannot have more than region name and servername");
}
// Arguments are regionname and an optional server name.
byte [] regionname = ((ImmutableBytesWritable)args[0]).get();
@@ -2001,8 +2057,8 @@ public class HMaster extends HasThread implements HMasterInterface,
if (hostnameAndPort == null) {
// Get server from the .META. if it wasn't passed as argument
hostnameAndPort =
- Bytes.toString(rr.getValue(HConstants.CATALOG_FAMILY,
- HConstants.SERVER_QUALIFIER));
+ Bytes.toString(rr.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.SERVER_QUALIFIER));
}
// Take region out of the intransistions in case it got stuck there doing
// an open or whatever.
@@ -2010,11 +2066,11 @@ public class HMaster extends HasThread implements HMasterInterface,
// If hostnameAndPort is still null, then none, exit.
if (hostnameAndPort == null) break;
long startCode =
- Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
- HConstants.STARTCODE_QUALIFIER));
+ Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.STARTCODE_QUALIFIER));
String name = HServerInfo.getServerName(hostnameAndPort, startCode);
LOG.info("Marking " + hri.getRegionNameAsString() +
- " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
+ " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
"master will tell regionserver to close region on next heartbeat");
this.regionManager.setClosing(name, hri, hri.isOffline());
break;
@@ -2053,14 +2109,14 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
static HRegionInfo getHRegionInfo(final byte[] row, final Result res)
- throws IOException {
+ throws IOException {
byte[] regioninfo = res.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
if (regioninfo == null) {
StringBuilder sb = new StringBuilder();
NavigableMap<byte[], byte[]> infoMap =
- res.getFamilyMap(HConstants.CATALOG_FAMILY);
+ res.getFamilyMap(HConstants.CATALOG_FAMILY);
if (infoMap == null) {
return null;
}
@@ -2133,12 +2189,12 @@ public class HMaster extends HasThread implements HMasterInterface,
final Configuration conf) {
try {
Constructor<? extends HMaster> c =
- masterClass.getConstructor(Configuration.class);
+ masterClass.getConstructor(Configuration.class);
return c.newInstance(conf);
} catch (Exception e) {
throw new RuntimeException("Failed construction of " +
- "Master: " + masterClass.toString() +
- ((e.getCause() != null)? e.getCause().getMessage(): ""), e);
+ "Master: " + masterClass.toString() +
+ ((e.getCause() != null)? e.getCause().getMessage(): ""), e);
}
}
@@ -2210,14 +2266,14 @@ public class HMaster extends HasThread implements HMasterInterface,
RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
if (runtime != null) {
LOG.info("vmName=" + runtime.getVmName() + ", vmVendor=" +
- runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion());
+ runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion());
LOG.info("vmInputArguments=" + runtime.getInputArguments());
}
// If 'local', defer to LocalHBaseCluster instance. Starts master
// and regionserver both in the one JVM.
if (LocalHBaseCluster.isLocal(conf)) {
final MiniZooKeeperCluster zooKeeperCluster =
- new MiniZooKeeperCluster();
+ new MiniZooKeeperCluster();
File zkDataPath = new File(conf.get("hbase.zookeeper.property.dataDir"));
int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
if (zkClientPort == 0) {
@@ -2234,11 +2290,11 @@ public class HMaster extends HasThread implements HMasterInterface,
throw new IOException(errorMsg);
}
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
- Integer.toString(clientPort));
+ Integer.toString(clientPort));
// Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out.
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
- LocalHMaster.class, HRegionServer.class);
+ LocalHMaster.class, HRegionServer.class);
((LocalHMaster)cluster.getMaster()).setZKCluster(zooKeeperCluster);
cluster.startup();
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 285db03..a68f1c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -59,7 +59,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -755,20 +754,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// clash w/ a store/family name. There is possibility, but assumption is
// that its slim (don't want to use control character in filename because
//
- Path regioninfo = new Path(this.regiondir, REGIONINFO_FILE);
- if (this.fs.exists(regioninfo) &&
- this.fs.getFileStatus(regioninfo).getLen() > 0) {
- return;
- }
- FSDataOutputStream out = this.fs.create(regioninfo, true);
- try {
- this.regionInfo.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(this.regionInfo.toString()));
- } finally {
- out.close();
- }
+ this.regionInfo.writeToDisk(conf, fs);
}
public AtomicLong getMemstoreSize() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
index 44738aa..1bbdd34 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
@@ -23,7 +23,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -45,7 +44,7 @@ import org.junit.experimental.categories.Category;
import java.io.IOException;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
/**
* Class to test the checkRegionInfo() and fixRegionInfo() methods of HBaseFsck.
@@ -80,17 +79,10 @@ public class TestHBaseFsckFixRegionInfo {
byte[] tableName = Bytes.toBytes("testFixRegionInfo");
- byte[][] splitKeys = {
- new byte[] { 1, 1, 1 },
- new byte[] { 2, 2, 2 },
- new byte[] { 3, 3, 3 },
- new byte[] { 4, 4, 4 },
- new byte[] { 5, 5, 5 },
- new byte[] { 6, 6, 6 },
- new byte[] { 7, 7, 7 },
- new byte[] { 8, 8, 8 },
- new byte[] { 9, 9, 9 },
- };
+ byte[][] splitKeys =
+ { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
+ new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
+ new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
@@ -100,79 +92,77 @@ public class TestHBaseFsckFixRegionInfo {
HBaseFsck fsckToCorrupt = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckToCorrupt.initAndScanRootMeta();
/*
- * The following line is necessary because some .regioninfo files (specifically the ROOT and the META)
- * become outdated immediately after initialization. This is because some parameters are immediately changed
- * after initialization, and these changes are not updated in the .regioninfo. So here we update all files.
- * This allows us to accurate measure the # of corrupted files later.
+ * The following line is necessary because some .regioninfo files (specifically the ROOT and the
+ * META) become outdated immediately after initialization. This is because some parameters are
+ * immediately changed after initialization, and these changes are not updated in the
+ * .regioninfo. So here we update all files. This allows us to accurate measure the # of
+ * corrupted files later.
*/
fsckToCorrupt.fixRegionInfo();
- //Randomly corrupt some files
+ // Randomly corrupt some files
int modifyCount = 0; // # of corrupted files
for (HbckInfo hbi : fsckToCorrupt.getRegionInfo().values()) {
if (Math.random() < 0.5) {
- Path tableDir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
- hbi.metaEntry.getTableDesc().getName());
+ Path tableDir =
+ HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), hbi.metaEntry.getTableDesc()
+ .getName());
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
- Path regionPath = HRegion.getRegionDir(tableDir,
- hbi.metaEntry.getEncodedName());
+ Path regionPath = HRegion.getRegionDir(tableDir, hbi.metaEntry.getEncodedName());
Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
- //read the original .regioninfo into an HRegionInfo
+ // read the original .regioninfo into an HRegionInfo
FSDataInputStream in = fs.open(regionInfoPath);
- HRegionInfo hri_original = new HRegionInfo();
- hri_original.readFields(in);
+ HRegionInfo hriOriginal = new HRegionInfo();
+ hriOriginal.readFields(in);
in.close();
- //change the HRegionInfo
- HRegionInfo hri_modified = randomlyModifyRegion(hri_original);
+ // change the HRegionInfo
+ HRegionInfo hriModified = randomlyModifyRegion(hriOriginal);
- //rewrite the original .regioninfo
- FSDataOutputStream out = fs.create(regionInfoPath, true);
- hri_modified.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(hri_modified.toString()));
- out.close();
+ // rewrite the original .regioninfo
+ hriModified.writeToDisk(conf);
+ System.out.println("SUCCESSFULLY CORRUPTED:"
+ + !hriOriginal.toString().equals(hriModified.toString()));
+ System.out.println("Original " + hriOriginal.toString());
+ System.out.println("Modified: " + hriModified.toString());
modifyCount++;
}
}
- //we incorrectly rewrote some .regioninfo files, so some .regioninfos are incorrect
+ // we incorrectly rewrote some .regioninfo files, so some .regioninfos are incorrect
HBaseFsck fsckCorrupted = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckCorrupted.initAndScanRootMeta();
Map<HRegionInfo, Path> risToRewrite = fsckCorrupted.checkRegionInfo();
- assertEquals(modifyCount, risToRewrite.size()); // # of files to rewrite must be the # of modified files
+ assertTrue("Expected " + modifyCount + " inconsistencies but saw " + risToRewrite.size()
+ + ".", risToRewrite.size() <= modifyCount); // # of files to rewrite is at most # of files modified
- //after fixing, we should see no errors
+ // after fixing, we should see no errors
HBaseFsck fsckFixed = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckFixed.initAndScanRootMeta();
fsckFixed.fixRegionInfo();
risToRewrite = fsckFixed.checkRegionInfo();
- assertEquals(0, risToRewrite.size());
+ assertEquals("Expected 0 inconsistencies after fixing but saw " + risToRewrite.size() + ".", 0,
+ risToRewrite.size());
}
- HRegionInfo randomlyModifyRegion(HRegionInfo hri){
- HTableDescriptor tableDesc = hri.getTableDesc();
+ HRegionInfo randomlyModifyRegion(HRegionInfo hri) {
+ HRegionInfo hriModified = new HRegionInfo(hri);
+ HTableDescriptor tableDescOriginal = hriModified.getTableDesc();
+ HTableDescriptor tableDescModified = new HTableDescriptor(tableDescOriginal);
double rand = Math.random();
- if (rand < 0.2){
- tableDesc.setReadOnly(!tableDesc.isReadOnly());
+ if (rand < 0.25) {
+ tableDescModified.setReadOnly(!tableDescOriginal.isReadOnly());
+ } else if (rand < 0.5) {
+ tableDescModified.setDeferredLogFlush(!tableDescOriginal.isDeferredLogFlush());
+ } else if (rand < 0.75) {
+ tableDescModified.setWALDisabled(!tableDescOriginal.isWALDisabled());
+ } else {
+ tableDescModified.setMemStoreFlushSize((long) tableDescOriginal.getMemStoreFlushSize() + 1); // is incorrect
}
- else if (rand < 0.4){
- tableDesc.setDeferredLogFlush(!tableDesc.isDeferredLogFlush());
- }
- else if (rand < 0.6){
- tableDesc.setName(Bytes.toBytes("NEWNAME"));
- }
- else if (rand < 0.8){
- tableDesc.setMaxFileSize( (long) 100);
- }
- else{
- tableDesc.setMemStoreFlushSize( (long) 200);
- }
- hri.setTableDesc(tableDesc);
- return hri;
+ hriModified.setTableDesc(tableDescModified);
+ return hriModified;
}
}
[20/50] [abbrv] git commit: [HBASE-11356] Escape regionName,
start/endKeys.
Posted by ec...@apache.org.
[HBASE-11356] Escape regionName, start/endKeys.
Summary: When using uniform splitter to create a table, the start/endKey and region names can contain characters that break HTML, like <j, or " .
Test Plan: No.
Reviewers: rshroff, elliott, manukranthk
Reviewed By: elliott, manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1384507
Tasks: 4517660
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42920 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af24ceab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af24ceab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af24ceab
Branch: refs/heads/0.89-fb
Commit: af24ceab6e9942a5127883b5ef540ac084fed148
Parents: 18a5653
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 17 19:32:37 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hbase/regionserver/HRegionServer.java | 27 +++----
.../hbase-webapps/master/assignmentPlan.jsp | 21 +++---
.../resources/hbase-webapps/master/table.jsp | 8 +--
.../hbase-webapps/regionserver/regionserver.jsp | 76 +++++++++++---------
4 files changed, 70 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/af24ceab/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7cbf606..7a5911b 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1583,20 +1583,21 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
* including region name, closeDate, startKey, endKey.
*/
public static class ClosedRegionInfo {
- private final String regionName;
+ private final String regionNameStr;
private final long closeDate;
- private final String startKey;
- private final String endKey;
+ private final String startKeyStr;
+ private final String endKeyStr;
- ClosedRegionInfo(String name, long date, byte[] startKey, byte[] endKey) {
- this.regionName = name;
+ ClosedRegionInfo(String regionNameStr, long date, byte[] startKey,
+ byte[] endKey) {
+ this.regionNameStr = regionNameStr;
this.closeDate = date;
- this.startKey = (startKey == null) ? "" : Bytes.toStringBinary(startKey);
- this.endKey = (endKey == null) ? "" : Bytes.toStringBinary(endKey);
+ this.startKeyStr = (startKey == null) ? "" : Bytes.toStringBinary(startKey);
+ this.endKeyStr = (endKey == null) ? "" : Bytes.toStringBinary(endKey);
}
- public String getRegionName() {
- return regionName;
+ public String getRegionNameAsString() {
+ return regionNameStr;
}
public long getCloseDate() {
@@ -1611,12 +1612,12 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
return formatter.format(date);
}
- public String getStartKey() {
- return startKey;
+ public String getStartKeyAsString() {
+ return startKeyStr;
}
- public String getEndKey() {
- return endKey;
+ public String getEndKeyAsString() {
+ return endKeyStr;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/af24ceab/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
index 8d23ba9..059cc28 100644
--- a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
+++ b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
@@ -1,21 +1,22 @@
<%@ page contentType="text/html;charset=UTF-8"
import="java.util.*"
+ import="org.apache.commons.lang.StringEscapeUtils"
import="org.apache.hadoop.conf.Configuration"
- import="org.apache.hadoop.hbase.util.Bytes"
- import="org.apache.hadoop.hbase.util.JvmVersion"
- import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.hbase.client.HBaseAdmin"
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.master.MetaRegion"
- import="org.apache.hadoop.hbase.client.HBaseAdmin"
- import="org.apache.hadoop.hbase.HServerInfo"
- import="org.apache.hadoop.hbase.HServerAddress"
- import="org.apache.hadoop.hbase.HTableDescriptor"
import="org.apache.hadoop.hbase.HColumnDescriptor"
import="org.apache.hadoop.hbase.HRegionInfo"
+ import="org.apache.hadoop.hbase.HServerAddress"
+ import="org.apache.hadoop.hbase.HServerInfo"
+ import="org.apache.hadoop.hbase.HTableDescriptor"
+ import="org.apache.hadoop.hbase.master.AssignmentPlan"
import="org.apache.hadoop.hbase.master.RegionPlacement"
import="org.apache.hadoop.hbase.master.RegionAssignmentSnapshot"
- import="org.apache.hadoop.hbase.master.AssignmentPlan"
+ import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.hbase.util.JvmVersion"
%><%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
Configuration conf = master.getConfiguration();
@@ -121,8 +122,8 @@
%>
<tr>
<form method="post">
- <input type="hidden" name="regionName" value="<%= regionName %>">
- <td><%=regionName%> </td>
+ <input type="hidden" name="regionName" value="<%= StringEscapeUtils.escapeHtml(regionName) %>">
+ <td><%= StringEscapeUtils.escapeHtml(regionName) %> </td>
<%
if (position.startsWith("Not")) {
%>
http://git-wip-us.apache.org/repos/asf/hbase/blob/af24ceab/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 704e43e..517ee37 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -1,9 +1,10 @@
<%@ page contentType="text/html;charset=UTF-8"
import="java.util.Map"
- import="org.apache.hadoop.io.Writable"
+ import="org.apache.commons.lang.StringEscapeUtils"
import="org.apache.hadoop.conf.Configuration"
import="org.apache.hadoop.hbase.client.HTable"
import="org.apache.hadoop.hbase.client.HBaseAdmin"
+ import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.HRegionInfo"
import="org.apache.hadoop.hbase.HServerAddress"
import="org.apache.hadoop.hbase.HServerInfo"
@@ -12,10 +13,9 @@
import="org.apache.hadoop.hbase.master.MetaRegion"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.io.Writable"
import="org.apache.hadoop.util.StringUtils"
- import="org.apache.commons.lang.StringEscapeUtils"
- import="java.util.Map"
- import="org.apache.hadoop.hbase.HConstants"%><%
+%><%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
Configuration conf = master.getConfiguration();
int rsInfoPort = conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/af24ceab/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/regionserver/regionserver.jsp b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
index c09b0bf..7d5c64b 100644
--- a/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
+++ b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
@@ -2,16 +2,18 @@
import="java.text.SimpleDateFormat"
import="java.util.*"
import="java.io.IOException"
+ import="org.apache.commons.lang.StringEscapeUtils"
import="org.apache.hadoop.io.Text"
+ import="org.apache.hadoop.hbase.HConstants"
+ import="org.apache.hadoop.hbase.HServerInfo"
+ import="org.apache.hadoop.hbase.HServerLoad"
+ import="org.apache.hadoop.hbase.HRegionInfo"
+ import="org.apache.hadoop.hbase.regionserver.HRegion"
import="org.apache.hadoop.hbase.regionserver.HRegionServer"
import="org.apache.hadoop.hbase.regionserver.HRegionServer.ClosedRegionInfo"
- import="org.apache.hadoop.hbase.regionserver.HRegion"
import="org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics"
import="org.apache.hadoop.hbase.util.Bytes"
- import="org.apache.hadoop.hbase.HConstants"
- import="org.apache.hadoop.hbase.HServerInfo"
- import="org.apache.hadoop.hbase.HServerLoad"
- import="org.apache.hadoop.hbase.HRegionInfo" %><%
+%><%
HRegionServer regionServer = (HRegionServer)getServletContext().getAttribute(HRegionServer.REGIONSERVER);
HServerInfo serverInfo = null;
try {
@@ -54,45 +56,49 @@
<h2>Online Regions</h2>
<% if (onlineRegionInfoAndOpenDate != null && onlineRegionInfoAndOpenDate.size() > 0) { %>
<table>
-<tr><th>Region Name</th>
- <th>Start Key</th>
- <th>End Key</th>
- <th>Metrics</th>
- <th>Open Time</th>
-</tr>
-<% for (HRegionInfo r : onlineRegionInfoAndOpenDate.keySet()) {
- HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
- %>
-
-<tr><td><%= r.getRegionNameAsString() %></td>
- <td><%= Bytes.toStringBinary(r.getStartKey()) %></td>
- <td><%= Bytes.toStringBinary(r.getEndKey()) %></td>
+ <tr><th>Region Name</th>
+ <th>Start Key</th>
+ <th>End Key</th>
+ <th>Metrics</th>
+ <th>Open Time</th>
+ </tr>
+<%
+ for (HRegionInfo r : onlineRegionInfoAndOpenDate.keySet()) {
+ HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
+%>
+ <tr>
+ <td><%= StringEscapeUtils.escapeHtml(r.getRegionNameAsString()) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(Bytes.toStringBinary(r.getStartKey())) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(Bytes.toStringBinary(r.getEndKey())) %></td>
<td><%= load.toString() %></td>
<td><%= onlineRegionInfoAndOpenDate.get(r) %></td>
-</tr>
+ </tr>
<%
}
%>
</table>
<h2>Recently Closed Regions</h2>
-<% if (recentlyClosedRegionInfo != null && recentlyClosedRegionInfo.size() > 0)
- { %>
+<%
+ if (recentlyClosedRegionInfo != null && recentlyClosedRegionInfo.size() > 0) {
+%>
<table>
-<tr><th>Region Name</th>
- <th>Start Key</th>
- <th>End Key</th>
- <th>Closed Time</th>
-</tr>
+ <tr>
+ <th>Region Name</th>
+ <th>Start Key</th>
+ <th>End Key</th>
+ <th>Closed Time</th>
+ </tr>
<%
- for (HRegionServer.ClosedRegionInfo info: recentlyClosedRegionInfo) {
+ for (HRegionServer.ClosedRegionInfo info: recentlyClosedRegionInfo) {
%>
-<tr><td><%= info.getRegionName() %></td>
- <td><%= info.getStartKey() %></td>
- <td><%= info.getEndKey() %></td>
- <td><%= info.getCloseDateAsString() %></td>
-</tr>
+ <tr>
+ <td><%= StringEscapeUtils.escapeHtml(info.getRegionNameAsString()) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(info.getStartKeyAsString()) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(info.getEndKeyAsString()) %></td>
+ <td><%= info.getCloseDateAsString() %></td>
+ </tr>
<%
- }
+ }
%>
</table>
<%
@@ -104,12 +110,12 @@ the region named
<em>domains,apache.org,5464829424211263407</em> is party to the table
<em>domains</em>, has an id of <em>5464829424211263407</em> and the first key
in the region is <em>apache.org</em>. The <em>-ROOT-</em>
-and <em>.META.</em> 'tables' are internal sytem tables (or 'catalog' tables in db-speak).
+and <em>.META.</em> 'tables' are internal system tables (or 'catalog' tables in db-speak).
The -ROOT- keeps a list of all regions in the .META. table. The .META. table
keeps a list of all regions in the system. The empty key is used to denote
table start and table end. A region with an empty start key is the first region in a table.
If region has both an empty start and an empty end key, its the only region in the table. See
-<a href="http://hbase.org">HBase Home</a> for further explication.<p>
+<a href="http://hbase.org">HBase Home</a> for further explanation.<p>
<% } else { %>
<p>Not serving regions</p>
<% } %>
[02/50] [abbrv] git commit: [master] Specify url of repo when
deploying automatically.
Posted by ec...@apache.org.
[master] Specify url of repo when deploying automatically.
Summary: Specify the repo url and name on the command line when deploying
Test Plan: http://nexus.vip.facebook.com:8181/nexus/content/repositories/libs-snapshots-local/com/facebook/hbase/hbase/3.0.42560-SNAPSHOT/
Reviewers: adela, oozie, jjmaestro
Reviewed By: jjmaestro
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1354446
Tasks: 4412611
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42574 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d016097
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d016097
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d016097
Branch: refs/heads/0.89-fb
Commit: 2d0160975c0477f063154aa058eba240577de471
Parents: 2c5c9ac
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri May 30 00:05:07 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:43:00 2014 -0700
----------------------------------------------------------------------
bin/devSupport/create_release.sh | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d016097/bin/devSupport/create_release.sh
----------------------------------------------------------------------
diff --git a/bin/devSupport/create_release.sh b/bin/devSupport/create_release.sh
index 6fd5707..75f9530 100755
--- a/bin/devSupport/create_release.sh
+++ b/bin/devSupport/create_release.sh
@@ -6,8 +6,24 @@ MVNCMD=${MVNCMD:-mvn}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
HBASE_HOME=${DIR}/../../
-VERSION=`${DIR}/generate_version.py --pom ${HBASE_HOME}/pom.xml --svn`
+if [ -d "${HBASE_HOME}/.git" ]; then
+ VERSION=`${DIR}/generate_version.py --pom ${HBASE_HOME}/pom.xml --git_svn`
+else
+ VERSION=`${DIR}/generate_version.py --pom ${HBASE_HOME}/pom.xml --svn`
+fi
+
echo $VERSION
+BASE_REPO=http://nexus.vip.facebook.com:8181/nexus/content/repositories
+
+if [[ $VERSION == *-SNAPSHOT* ]]; then
+ REPO_NAME=libs-snapshots-local
+else
+ REPO_NAME=libs-releases-local
+fi
+
+REPO_URL=${BASE_REPO}/${REPO_NAME}
+
+
$MVNCMD versions:set -DnewVersion=${VERSION}
-$MVNCMD clean package assembly:single deploy -DskipTests
\ No newline at end of file
+$MVNCMD clean package assembly:single deploy -DskipTests -DaltDeploymentRepository=${REPO_NAME}::default::${REPO_URL}
[06/50] [abbrv] git commit: [master] Pull native libs from internal
jars.
Posted by ec...@apache.org.
[master] Pull native libs from internal jars.
Summary: .so's are now published internally. We should pull from there and un-jar at runtime.
Test Plan: deploy on dev038 and see that lzo works.
Reviewers: oozie, jjmaestro, adela, pwhite
Reviewed By: pwhite
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1357483
Tasks: 4427997
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42651 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Conflicts:
.gitignore
lib/native/Linux-amd64-64/libhadoop.a
lib/native/Linux-amd64-64/libhadoop.la
lib/native/Linux-amd64-64/libhadoop.so
lib/native/Linux-amd64-64/libhadoop.so.1
lib/native/Linux-amd64-64/libhadoop.so.1.0.0
lib/native/Linux-amd64-64/libhadoopsnappy.a
lib/native/Linux-amd64-64/libhadoopsnappy.la
lib/native/Linux-amd64-64/libhadoopsnappy.lai
lib/native/Linux-amd64-64/libhadoopsnappy.so
lib/native/Linux-amd64-64/libhadoopsnappy.so.0
lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a404f16c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a404f16c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a404f16c
Branch: refs/heads/0.89-fb
Commit: a404f16c9c592c5581e86b6f4eba9b2f128a30ef
Parents: 4f7a7ca
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 3 21:29:46 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:02 2014 -0700
----------------------------------------------------------------------
.gitignore | 3 +-
bin/hbase | 30 ++++++++++++++
lib/native/Linux-amd64-64/libhadoop.a | Bin 110792 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.la | 35 ----------------
lib/native/Linux-amd64-64/libhadoop.so | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.so.1 | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.so.1.0.0 | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.a | Bin 86232 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.la | 35 ----------------
lib/native/Linux-amd64-64/libhadoopsnappy.lai | 35 ----------------
lib/native/Linux-amd64-64/libhadoopsnappy.so | Bin 55233 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.so.0 | Bin 55233 -> 0 bytes
.../Linux-amd64-64/libhadoopsnappy.so.0.0.1 | Bin 55233 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.a | Bin 310360 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.la | 41 -------------------
lib/native/Linux-amd64-64/libsnappy.so | Bin 151171 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.so.1 | Bin 151171 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.so.1.1.3 | Bin 151171 -> 0 bytes
pom.xml | 35 +++++++++++++++-
19 files changed, 66 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 42f6eaa..3c4afc6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,8 @@
/.project
/.settings
/build
-/lib/native/Linux-amd64-64/libgpl*
+/lib/native/
+/lib/hadoop-lzo-0.4.1.jar
/.idea/
/logs
/target
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 43e2472..2834729 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -152,6 +152,34 @@ add_maven_test_classes_to_classpath() {
fi
}
+expand_native_jars() {
+ native_dir=$HBASE_HOME/lib/native/
+ if [ -d ${native_dir} ]; then
+ return;
+ fi
+
+ mkdir -p "${native_dir}"
+
+ while read -d: jarname; do
+ if [[ ${jarname} == *-native*.jar ]] && [ -f ${jarname} ]; then
+ expand_single_native_jar ${jarname} ${native_dir}
+ fi
+ done <<< $1
+}
+
+expand_single_native_jar() {
+ jar=$1
+ target=$2
+ TEMPDIR=`mktemp --directory -t hbase-native.XXXXXXX 2>/dev/null || mktemp -d -t 'hbase-native'`
+ pushd $TEMPDIR
+ unzip $jar
+ for nd in $(find . -name "native" -type d); do
+ cp -r ${nd}/* ${target}
+ done
+ popd
+ rm -rf $TEMPDIR
+}
+
# Add maven target directory
if $in_sources_dir; then
add_maven_deps_to_classpath
@@ -227,6 +255,8 @@ fi
# restore ordinary behaviour
unset IFS
+expand_native_jars $CLASSPATH
+
# figure out which class to run
if [ "$COMMAND" = "shell" ] ; then
CLASS="org.jruby.Main -X+O ${HBASE_HOME}/bin/hirb.rb"
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.a b/lib/native/Linux-amd64-64/libhadoop.a
deleted file mode 100644
index 8ab6256..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.la b/lib/native/Linux-amd64-64/libhadoop.la
deleted file mode 100644
index 49d8da2..0000000
--- a/lib/native/Linux-amd64-64/libhadoop.la
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoop.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoop.so.1'
-
-# Names of this library.
-library_names='libhadoop.so.1.0.0 libhadoop.so.1 libhadoop.so'
-
-# The name of the static archive.
-old_library='libhadoop.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -ljvm -ldl'
-
-# Version information for libhadoop.
-current=1
-age=0
-revision=0
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so b/lib/native/Linux-amd64-64/libhadoop.so
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so.1 b/lib/native/Linux-amd64-64/libhadoop.so.1
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 b/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.a b/lib/native/Linux-amd64-64/libhadoopsnappy.a
deleted file mode 100644
index 1d6e9b6..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.la b/lib/native/Linux-amd64-64/libhadoopsnappy.la
deleted file mode 100644
index 1c81f2e..0000000
--- a/lib/native/Linux-amd64-64/libhadoopsnappy.la
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoopsnappy.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoopsnappy.so.0'
-
-# Names of this library.
-library_names='libhadoopsnappy.so.0.0.1 libhadoopsnappy.so.0 libhadoopsnappy.so'
-
-# The name of the static archive.
-old_library='libhadoopsnappy.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -L/data/users/feifei/titan-git-svn/VENDOR/hadoop-0.20/src/native/lib/snappy/lib -ljvm -ldl'
-
-# Version information for libhadoopsnappy.
-current=0
-age=0
-revision=1
-
-# Is this an already installed library?
-installed=no
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.lai
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.lai b/lib/native/Linux-amd64-64/libhadoopsnappy.lai
deleted file mode 100644
index 6d622cb..0000000
--- a/lib/native/Linux-amd64-64/libhadoopsnappy.lai
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoopsnappy.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoopsnappy.so.0'
-
-# Names of this library.
-library_names='libhadoopsnappy.so.0.0.1 libhadoopsnappy.so.0 libhadoopsnappy.so'
-
-# The name of the static archive.
-old_library='libhadoopsnappy.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -L/data/users/feifei/titan-git-svn/VENDOR/hadoop-0.20/src/native/lib/snappy/lib -ljvm -ldl'
-
-# Version information for libhadoopsnappy.
-current=0
-age=0
-revision=1
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so b/lib/native/Linux-amd64-64/libhadoopsnappy.so
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so.0
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0 b/lib/native/Linux-amd64-64/libhadoopsnappy.so.0
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1 b/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.a b/lib/native/Linux-amd64-64/libsnappy.a
deleted file mode 100644
index 16cfca9..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.la b/lib/native/Linux-amd64-64/libsnappy.la
deleted file mode 100755
index d45a9b5..0000000
--- a/lib/native/Linux-amd64-64/libsnappy.la
+++ /dev/null
@@ -1,41 +0,0 @@
-# libsnappy.la - a libtool library file
-# Generated by ltmain.sh (GNU libtool) 2.2.6b Debian-2.2.6b-2ubuntu1
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libsnappy.so.1'
-
-# Names of this library.
-library_names='libsnappy.so.1.1.3 libsnappy.so.1 libsnappy.so'
-
-# The name of the static archive.
-old_library='libsnappy.a'
-
-# Linker flags that can not go in dependency_libs.
-inherited_linker_flags=''
-
-# Libraries that this one depends upon.
-dependency_libs=''
-
-# Names of additional weak libraries provided by this library
-weak_library_names=''
-
-# Version information for libsnappy.
-current=2
-age=1
-revision=3
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so b/lib/native/Linux-amd64-64/libsnappy.so
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so.1 b/lib/native/Linux-amd64-64/libsnappy.so.1
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so.1.1.3
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so.1.1.3 b/lib/native/Linux-amd64-64/libsnappy.so.1.1.3
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so.1.1.3 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 70fbee7..02e8ea3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -658,7 +658,7 @@
<compileSource>1.7</compileSource>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hbase.version>0.21.0-SNAPSHOT</hbase.version>
- <hadoop.version>4.42343</hadoop.version>
+ <hadoop.version>4.42426</hadoop.version>
<swift.version>0.12.0</swift.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-io.version>2.1</commons-io.version>
@@ -722,6 +722,22 @@
</dependency>
<dependency>
<groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-lzo</artifactId>
+ <classifier>native</classifier>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-snappy-native</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-native</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
@@ -1010,6 +1026,23 @@
<scope>runtime</scope>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-lzo</artifactId>
+ <classifier>native</classifier>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-snappy-native</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-native</artifactId>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
[28/50] [abbrv] git commit: [master] Change configuration of
observers also to support project@version format
Posted by ec...@apache.org.
[master] Change configuration of observers also to support project@version format
Summary:
After this diff each entry is in the following format:
project@version->table[,table]
or
className->table[,table]
Test Plan:
`TestClassLoading`
`TestEndpointReload`
Reviewers: manukranthk, adela
Reviewed By: adela
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1352561
Tasks: 4414918
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43112 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a45454db
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a45454db
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a45454db
Branch: refs/heads/0.89-fb
Commit: a45454db37e1a663a42674a083d69911cfc7aee2
Parents: f0eccb5
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 19:57:37 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hbase/coprocessor/CoprocessorHost.java | 199 +++++++++++++++----
.../coprocessor/RegionCoprocessorHost.java | 6 +-
.../coprocessor/endpoints/EndpointLoader.java | 101 ++--------
.../endpoints/TestEndpointReload.java | 4 +-
.../TestHRegionObserverBypassCoprocessor.java | 6 +-
.../util/coprocessor/TestClassLoading.java | 91 +++++----
6 files changed, 237 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 6e98f57..b04a57a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -31,17 +31,22 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.ipc.ThriftClientInterface;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.util.coprocessor.CoprocessorClassLoader;
@@ -89,6 +94,8 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
// unique file prefix to use for local copies of jars when classloading
protected String pathPrefix;
protected AtomicInteger loadSequence = new AtomicInteger();
+ public static final Pattern PAT_PROJECT_VERSION =
+ Pattern.compile("[^@]+@[0-9]+");
/**
* The field name for "loaded classes"
*/
@@ -204,14 +211,15 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
/**
* Read coprocessors that should be loaded from configuration. In the
- * configuration we should have string of tuples of [table, jar, class] for
- * each usage of coprocesssor
+ * configuration we should have string of tuples of
+ * "project@version->table[,table]" or "classname->table[,table]"
+ * for each usage of coprocessor
*
* @param conf
*/
- public Map<String, List<Pair<String, String>>> readCoprocessorsFromConf(
+ public Map<String, List<String>> readCoprocessorsFromConf(
Configuration conf) {
- Map<String, List<Pair<String, String>>> coprocConfigMap = new HashMap<>();
+ Map<String, List<String>> coprocConfigMap = new HashMap<>();
String fromConfig = conf.get(USER_REGION_COPROCESSOR_FROM_HDFS_KEY);
if (fromConfig == null || fromConfig.isEmpty()) {
return coprocConfigMap;
@@ -219,27 +227,29 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
String[] tuples = fromConfig.split(";");
for (String tuple : tuples) {
tuple = tuple.trim();
- String[] str = tuple.split(",");
- // str[0] is table, str[1] is path for the jar, str[2] is name of the
- // class
- if (str.length != 3) {
- LOG.error("Configuration is not in expected format: " + str);
- return null;
+ String[] pathAndTables = tuple.split("->", 2);
+ if (pathAndTables.length != 2) {
+ LOG.error("Configuration is not in expected format: " + tuple);
+ // Ignore this tuple
+ continue;
}
- String table = str[0].trim();
- List<Pair<String, String>> inMap = coprocConfigMap.get(table);
- if (inMap == null) {
- inMap = new ArrayList<>();
- coprocConfigMap.put(table, inMap);
+ String path = pathAndTables[0].trim();
+ String[] tables = pathAndTables[1].trim().split(",");
+ for (String table : tables) {
+ List<String> inMap = coprocConfigMap.get(table);
+ if (inMap == null) {
+ inMap = new ArrayList<>();
+ coprocConfigMap.put(table, inMap);
+ }
+ inMap.add(path);
}
- inMap.add(new Pair<>(str[1].trim(), str[2].trim()));
}
return coprocConfigMap;
}
/**
* Checks if the specified path for the coprocessor jar is in expected format
- * TODO: make this more advanced - currently is very hardoced and dummy
+ * TODO: make this more advanced - currently is very hardcoded and dummy
*
* @param path
* - Absolute and complete path where the coprocessor jar resides
@@ -261,25 +271,23 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
}
/**
- * Used to load coprocessors whose jar is specified via hdfs path. All
+ * Used to load coprocessors whose jar is specified via HDFS path. All
* existing coprocessors will be unloaded (if they appear again in the new
* configuration then they will be re-added).
*
* @param config
* @throws IOException
*/
- protected void reloadCoprocessorsFromHdfs(String table,
- List<Pair<String, String>> toLoad, Configuration config)
- throws IOException {
+ protected void reloadCoprocessorsFromHdfs(String table, List<String> toLoad,
+ Configuration config) throws IOException {
// remove whatever is loaded first
removeLoadedCoprocWithKey(USER_REGION_COPROCESSOR_FROM_HDFS_KEY);
List<E> newCoprocessors = new ArrayList<>();
LOG.info("Loading coprocessor for table: " + table);
- for (Pair<String, String> pair : toLoad) {
- LOG.info("Jar: " + pair.getFirst() + " class: " + pair.getSecond());
- E coproc = load(new Path(pair.getFirst()), pair.getSecond(),
- Coprocessor.PRIORITY_USER, config,
+ for (String path : toLoad) {
+ LOG.info("Loading from " + path);
+ E coproc = load(path, Coprocessor.PRIORITY_USER, config,
USER_REGION_COPROCESSOR_FROM_HDFS_KEY);
newCoprocessors.add(coproc);
}
@@ -300,38 +308,48 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
* @param conf configuration for coprocessor
* @throws java.io.IOException Exception
*/
- public E load(Path path, String className, int priority,
- Configuration conf, String confKey) throws IOException {
+ public E load(String path, int priority, Configuration conf, String confKey)
+ throws IOException {
Class<?> implClass = null;
- LOG.debug("Loading coprocessor class " + className + " with path " +
- path + " and priority " + priority);
-
ClassLoader cl = null;
- if (path == null) {
- try {
- implClass = getClass().getClassLoader().loadClass(className);
- } catch (ClassNotFoundException e) {
- throw new IOException("No jar path specified for " + className);
+ if (PAT_PROJECT_VERSION.matcher(path).matches()) {
+ String[] projVer = path.split("@");
+ String project = projVer[0];
+ int version = Integer.parseInt(projVer[1]);
+
+ FileSystem fs = FileSystem.get(conf);
+ String dfsRoot = getCoprocessorDfsRoot(conf);
+
+ Pair<List<Class<?>>, ClassLoader> classesAndLoader =
+ getProjectClassesAndLoader(conf, fs, dfsRoot, project, version,
+ pathPrefix);
+ if (classesAndLoader.getFirst().size() != 1) {
+ // The number of loaded classes should be one for observers.
+ throw new IOException("The number of loaded classes for observers "
+ + "should be one but we got: " + classesAndLoader.getFirst());
}
+ implClass = classesAndLoader.getFirst().get(0);
+ cl = classesAndLoader.getSecond();
} else {
- cl = CoprocessorClassLoader.getClassLoader(
- path, getClass().getClassLoader(), pathPrefix, conf);
+ String className = path;
+ LOG.debug("Loading coprocessor class " + className + " with path " + path
+ + " and priority " + priority);
try {
+ cl = this.getClass().getClassLoader();
implClass = cl.loadClass(className);
} catch (ClassNotFoundException e) {
- throw new IOException("Cannot load external coprocessor class " + className, e);
+ throw new IOException("Cannot load coprocessor class " + className, e);
}
}
+
//load custom code for coprocessor
try (ContextResetter ctxResetter = new ContextResetter(cl)) {
-// switch temporarily to the thread classloader for custom CP
+ // switch temporarily to the thread classloader for custom CP
E cpInstance = loadInstance(implClass, priority, conf, confKey);
return cpInstance;
} catch (Exception e) {
- String msg = new StringBuilder()
- .append("Cannot load external coprocessor class ").append(className)
- .toString();
- throw new IOException(msg, e);
+ throw new IOException("Cannot load external coprocessor class "
+ + implClass.getName(), e);
}
}
@@ -634,6 +652,101 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
}
}
+ public static ClassLoader getDfsClassLoader(Configuration conf,
+ FileSystem fs, Path projectPath, String localPathPrefx)
+ throws IOException {
+ FileStatus[] jars = fs.listStatus(projectPath, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().endsWith(".jar");
+ }
+ });
+
+ if (jars.length < 1) {
+ throw new IOException("No jars in " + projectPath + " found!");
+ }
+
+ // TODO load class based on all jars.
+ return CoprocessorClassLoader.getClassLoader(jars[0].getPath(),
+ CoprocessorHost.class.getClassLoader(), localPathPrefx, conf);
+ }
+
+ /**
+ * Returns an entry string for a version of a project.
+ */
+ public static String genDfsProjectEntry(String project, int version) {
+ return project + "@" + version;
+ }
+
+ /**
+ * Returns an entry string for a version of a project for a table.
+ */
+ public static String genDfsPerTableProjectEntry(String project, int version,
+ String tableName) {
+ return project + "@" + version + "->" + tableName;
+ }
+
+ /**
+ * Returns the list of classes and ClassLoader to load them.
+ *
+ * @param conf The configuration
+ * @param fs The file-system.
+ * @param dfsRootPath The root path of coprocessors on DFS.
+ * @param project The name of the project.
+ * @param version The version of the project to load
+ * @param localPathPrefx The prefix of paths for CoprocessorClassLoader to
+ * copy jars to local.
+ * @return A Pair of classes to load and the ClassLoader.
+ */
+ @SuppressWarnings({ "unchecked" })
+ public static Pair<List<Class<?>>, ClassLoader> getProjectClassesAndLoader(
+ Configuration conf, FileSystem fs,
+ String dfsRootPath, String project, int version, String localPathPrefx)
+ throws IOException {
+ Path projectPath =
+ new Path(getCoprocessorPath(dfsRootPath, project, version));
+ if (!fs.exists(projectPath)) {
+ throw new IOException("Folder " + projectPath + " doesn't exist!");
+ }
+ if (!fs.getFileStatus(projectPath).isDir()) {
+ throw new IOException(projectPath + " is not a folder!");
+ }
+
+ Path configPath = new Path(projectPath, CONFIG_JSON);
+ if (!fs.exists(configPath)) {
+ throw new IOException("Cannot find config file: " + configPath);
+ }
+
+ Map<String, Object> jConf = FSUtils.readJSONFromFile(fs, configPath);
+ String name = (String) jConf.get(COPROCESSOR_JSON_NAME_FIELD);
+ if (!name.equals(project)) {
+ throw new IOException("Loaded name " + name + " is not expected"
+ + project);
+ }
+
+ int loadedVer = (int) jConf.get(COPROCESSOR_JSON_VERSION_FIELD);
+ if (loadedVer != version) {
+ throw new IOException("Loaded version " + loadedVer + " is not expected"
+ + version);
+ }
+
+ ArrayList<String> loadedClasses =
+ (ArrayList<String>) jConf.get(COPROCESSOR_JSON_LOADED_CLASSES_FIELD);
+
+ ClassLoader cl = getDfsClassLoader(conf, fs, projectPath, localPathPrefx);
+ List<Class<?>> res = new ArrayList<>(loadedClasses.size());
+ for (int i = 0; i < loadedClasses.size(); i++) {
+ try {
+ Class<?> cls = cl.loadClass(loadedClasses.get(i));
+ res.add(cls);
+ } catch (Throwable t) {
+ LOG.error("Load class " + loadedClasses.get(i) + " failed, ignored!", t);
+ }
+ }
+
+ return Pair.newPair(res, cl);
+ }
+
/**
* Used just to set the contextClassLoader on the current thread in case of
* exception - code in {@link #close()} will be executed
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorHost.java
index f39a765..f000099 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorHost.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.coprocessor.observers.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.observers.RegionObserver;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Pair;
/**
* Implements the coprocessor environment and runtime support for coprocessors
@@ -80,10 +79,10 @@ public class RegionCoprocessorHost extends
* @throws IOException
*/
public void reloadCoprocessors(Configuration newConf) throws IOException {
- Map<String, List<Pair<String, String>>> map = this
+ Map<String, List<String>> map = this
.readCoprocessorsFromConf(newConf);
String coprocTable = this.region.getTableDesc().getNameAsString();
- List<Pair<String, String>> forTable = map.get(coprocTable);
+ List<String> forTable = map.get(coprocTable);
// reload system default cp's from configuration.
reloadSysCoprocessorsOnConfigChange(newConf, REGION_COPROCESSOR_CONF_KEY);
// reload system default cp's for user tables from configuration.
@@ -216,6 +215,7 @@ public class RegionCoprocessorHost extends
}
+ @Override
public void shutdown() {
super.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/EndpointLoader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/EndpointLoader.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/EndpointLoader.java
index 794f91b..55c87e0 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/EndpointLoader.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/EndpointLoader.java
@@ -22,20 +22,14 @@ package org.apache.hadoop.hbase.coprocessor.endpoints;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
-import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.coprocessor.CoprocessorClassLoader;
+import org.apache.hadoop.hbase.util.Pair;
/**
* Static methods for loading endpoints.
@@ -55,86 +49,29 @@ public class EndpointLoader {
private static final String dfsTmpPrefix = "endpoint."
+ UUID.randomUUID().toString();
- private static final Pattern PAT_PROJECT_VERSION =
- Pattern.compile("[^@]+@[0-9]+");
-
- /**
- * Returns an entry string for a version of a project.
- */
- public static String genDfsEndpointEntry(String project, int version) {
- return project + "@" + version;
- }
-
- private static ClassLoader getDfsClassLoader(Configuration conf,
- FileSystem fs, Path projectPath) throws IOException {
- FileStatus[] jars = fs.listStatus(projectPath, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(".jar");
- }
- });
-
- if (jars.length < 1) {
- throw new IOException("No jars in " + projectPath + " found!");
- }
-
- // TODO load class based on all jars.
- return CoprocessorClassLoader.getClassLoader(jars[0].getPath(),
- EndpointLoader.class.getClassLoader(), dfsTmpPrefix, conf);
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private static List<IEndpointFactory> loadProject(Configuration conf,
+ @SuppressWarnings({ "rawtypes" })
+ private static List<IEndpointFactory> loadEndpointProject(Configuration conf,
FileSystem fs, String dfsRootPath, String project, int version)
throws IOException {
- Path projectPath = new Path(CoprocessorHost.getCoprocessorPath(dfsRootPath,
- project, version));
- if (!fs.exists(projectPath)) {
- throw new IOException("Folder " + projectPath + " doesn't exist!");
- }
- if (!fs.getFileStatus(projectPath).isDir()) {
- throw new IOException(projectPath + " is not a folder!");
- }
-
- Path configPath = new Path(projectPath, CoprocessorHost.CONFIG_JSON);
- if (!fs.exists(configPath)) {
- throw new IOException("Cannot find config file: " + configPath);
- }
-
- Map<String, Object> jConf = FSUtils.readJSONFromFile(fs, configPath);
- String name = (String) jConf.get(
- CoprocessorHost.COPROCESSOR_JSON_NAME_FIELD);
- if (!name.equals(project)) {
- throw new IOException("Loaded name " + name + " is not expected"
- + project);
- }
- int loadedVer =
- (int) jConf.get(CoprocessorHost.COPROCESSOR_JSON_VERSION_FIELD);
- if (loadedVer != version) {
- throw new IOException("Loaded version " + loadedVer + " is not expected"
- + version);
- }
-
- ArrayList<String> loadedClasses = (ArrayList<String>) jConf.get(
- CoprocessorHost.COPROCESSOR_JSON_LOADED_CLASSES_FIELD);
-
- ClassLoader cl = getDfsClassLoader(conf, fs, projectPath);
- List<IEndpointFactory> res = new ArrayList<>(loadedClasses.size());
- for (int i = 0; i < loadedClasses.size(); i++) {
+ Pair<List<Class<?>>, ClassLoader> classesAndLoader =
+ CoprocessorHost.getProjectClassesAndLoader(conf, fs, dfsRootPath, project, version,
+ dfsTmpPrefix);
+
+ List<IEndpointFactory> res = new ArrayList<>(
+ classesAndLoader.getFirst().size());
+ for (Class<?> cls: classesAndLoader.getFirst()) {
+ if (!IEndpointFactory.class.isAssignableFrom(cls)) {
+ LOG.info("Class " + cls + " cannot be assigned as "
+ + IEndpointFactory.class + ", ignored!");
+ continue;
+ }
try {
- Class<?> cls = cl.loadClass(loadedClasses.get(i));
- if (!IEndpointFactory.class.isAssignableFrom(cls)) {
- LOG.info("Class " + cls + " cannot be assigned as "
- + IEndpointFactory.class + ", ignored!");
- continue;
- }
res.add((IEndpointFactory) cls.newInstance());
} catch (Throwable t) {
- LOG.error("Load class " + loadedClasses.get(i) + " failed!", t);
+ LOG.info("Creating instance of " + cls + " failed!", t);
}
}
-
return res;
}
@@ -154,7 +91,7 @@ public class EndpointLoader {
for (String className : classNames) {
try {
- if (PAT_PROJECT_VERSION.matcher(className).matches()) {
+ if (CoprocessorHost.PAT_PROJECT_VERSION.matcher(className).matches()) {
String[] projVer = className.split("@");
String project = projVer[0];
int version = Integer.parseInt(projVer[1]);
@@ -162,8 +99,8 @@ public class EndpointLoader {
if (fs == null) {
fs = FileSystem.get(conf);
}
- List<IEndpointFactory> factories = loadProject(conf, fs, dfsRoot,
- project, version);
+ List<IEndpointFactory> factories = loadEndpointProject(conf, fs,
+ dfsRoot, project, version);
for (IEndpointFactory factory : factories) {
LOG.info("Loading endpoint class " + factory.getClass().getName());
reg.register(factory.getEndpointInterface(), factory);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/test/java/org/apache/hadoop/hbase/coprocessor/endpoints/TestEndpointReload.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/endpoints/TestEndpointReload.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/endpoints/TestEndpointReload.java
index 872758b..58310d0 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/endpoints/TestEndpointReload.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/endpoints/TestEndpointReload.java
@@ -244,14 +244,14 @@ public class TestEndpointReload {
// two
conf.setStrings(EndpointLoader.FACTORY_CLASSES_KEY,
- EndpointLoader.genDfsEndpointEntry(TWO_NAME, TWO_VERSION));
+ CoprocessorHost.genDfsProjectEntry(TWO_NAME, TWO_VERSION));
HRegionServer.configurationManager.notifyAllObservers(conf);
checkEndpionts(cp, jarFile, false, true);
// one+two
conf.setStrings(EndpointLoader.FACTORY_CLASSES_KEY,
OneFactory.class.getName(),
- EndpointLoader.genDfsEndpointEntry(TWO_NAME, TWO_VERSION));
+ CoprocessorHost.genDfsProjectEntry(TWO_NAME, TWO_VERSION));
HRegionServer.configurationManager.notifyAllObservers(conf);
checkEndpionts(cp, jarFile, true, true);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/test/java/org/apache/hadoop/hbase/coprocessor/observers/TestHRegionObserverBypassCoprocessor.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/observers/TestHRegionObserverBypassCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/observers/TestHRegionObserverBypassCoprocessor.java
index 08a2e20..ce230ae 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/observers/TestHRegionObserverBypassCoprocessor.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/observers/TestHRegionObserverBypassCoprocessor.java
@@ -134,9 +134,9 @@ public class TestHRegionObserverBypassCoprocessor {
HTable table = new HTable(util.getConfiguration(), tableName);
Set<String> allCoprocessors = RegionCoprocessorHost
.getEverLoadedCoprocessors();
- Assert.assertEquals("There should be only one coprocessor everloaded", 1, allCoprocessors.size());
- Assert
- .assertEquals(
+ Assert.assertEquals("There should be only one coprocessor everloaded", 1,
+ allCoprocessors.size());
+ Assert.assertEquals(
"Expected loaded coprocessor is different from one which is currently loaded",
TestCoprocessor.class.getName(), allCoprocessors.toArray()[0]);
Configuration conf = util.getConfiguration();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a45454db/src/test/java/org/apache/hadoop/hbase/util/coprocessor/TestClassLoading.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/util/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/util/coprocessor/TestClassLoading.java
index b787b60..569c663 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/coprocessor/TestClassLoading.java
@@ -19,27 +19,28 @@
package org.apache.hadoop.hbase.util.coprocessor;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.io.OutputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorAdmin;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.observers.TestHRegionObserverBypassCoprocessor.TestCoprocessor;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -49,13 +50,10 @@ import org.junit.Test;
* Test coprocessors class loading.
*/
public class TestClassLoading {
- private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final byte[] DUMMY = Bytes.toBytes("dummy");
private static final byte[] TEST = Bytes.toBytes("test");
- private static MiniDFSCluster cluster;
-
static final String tableName = "TestClassLoading";
static final String cpName1 = "TestCP1";
static final String cpName2 = "TestCP2";
@@ -69,7 +67,6 @@ public class TestClassLoading {
conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
testCoprocessor.getName());
TEST_UTIL.startMiniCluster(1);
- cluster = TEST_UTIL.getDFSCluster();
}
@AfterClass
@@ -104,38 +101,58 @@ public class TestClassLoading {
return newStr;
}
+ private void uploadCoprocessor(File jarFile, String name, int version,
+ String className) throws Exception {
+ Path localPath =
+ TEST_UTIL.getTestDir("TestClassLoading-uploadCoprocessor-" + name
+ + "-" + version);
+
+ LocalFileSystem lfs = FileSystem.getLocal(TEST_UTIL.getConfiguration());
+ lfs.delete(localPath, true);
+
+ lfs.mkdirs(localPath);
+ lfs.copyFromLocalFile(false, new Path(jarFile.toString()), new Path(
+ localPath, jarFile.getName()));
+
+ JsonFactory f = new JsonFactory();
+ try (OutputStream out =
+ lfs.create(new Path(localPath, CoprocessorHost.CONFIG_JSON))) {
+ JsonGenerator g = f.createJsonGenerator(out);
+ g.writeStartObject();
+
+ g.writeStringField(CoprocessorHost.COPROCESSOR_JSON_NAME_FIELD, name);
+
+ g.writeNumberField(CoprocessorHost.COPROCESSOR_JSON_VERSION_FIELD,
+ version);
+
+ g.writeArrayFieldStart(CoprocessorHost.COPROCESSOR_JSON_LOADED_CLASSES_FIELD);
+ g.writeString(className);
+ g.writeEndArray();
+ g.writeEndObject();
+ g.close();
+ }
+
+ new CoprocessorAdmin(TEST_UTIL.getConfiguration()).upload(localPath, true);
+ }
+
@Test
public void testClassLoadingFromHDFS() throws Exception {
- FileSystem fs = cluster.getFileSystem();
+ final String CP_NAME = "test";
+
+ // Create jar files.
File jarFile1 = buildCoprocessorJar(cpName1);
System.out.println(jarFile1.getName());
File jarFile2 = buildCoprocessorJar(cpName2);
- // have to create directories because we are not placing the files on hdfs
- // root
- assertTrue(fs.mkdirs(new Path("/coprocessors/test/1")));
- // copy the jars into dfs
- fs.moveFromLocalFile(new Path(jarFile1.getPath()), new Path(fs.getUri()
- .toString(), buildCorrectPathForCoprocessorJar("") + Path.SEPARATOR));
-
- String jarFileOnHDFS1 = buildCorrectPathForCoprocessorJar(fs.getUri()
- .toString()) + Path.SEPARATOR + jarFile1.getName();
- Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
- System.out.println(jarFileOnHDFS1);
- assertTrue("Copy jar file to HDFS failed.", fs.exists(pathOnHDFS1));
- LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
-
- fs.moveFromLocalFile(new Path(jarFile2.getPath()), new Path(fs.getUri()
- .toString(), buildCorrectPathForCoprocessorJar("") + Path.SEPARATOR));
-
- String jarFileOnHDFS2 = buildCorrectPathForCoprocessorJar(fs.getUri()
- .toString()) + Path.SEPARATOR + jarFile2.getName();
- Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
- assertTrue("Copy jar file to HDFS failed.", fs.exists(pathOnHDFS2));
- LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
+ System.out.println(jarFile2.getName());
+ // Upload jars to dfs
+ uploadCoprocessor(jarFile1, CP_NAME, 1, cpName1);
+ uploadCoprocessor(jarFile2, CP_NAME, 2, cpName2);
+
Configuration conf = TEST_UTIL.getConfiguration();
// check if only TestCoprocessor is currently loaded
- List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(Bytes.toBytes(tableName));
+ List<HRegion> regions =
+ TEST_UTIL.getMiniHBaseCluster().getRegions(Bytes.toBytes(tableName));
Set<String> expectedCoprocessorSimpleName = new HashSet<>();
Set<String> allCoprocessors = RegionCoprocessorHost
.getEverLoadedCoprocessors();
@@ -148,7 +165,7 @@ public class TestClassLoading {
// remove the firstly added coprocessor
conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, "");
conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_FROM_HDFS_KEY,
- tableName + "," + pathOnHDFS1.toString() + "," + cpName1);
+ CoprocessorHost.genDfsPerTableProjectEntry(CP_NAME, 1, tableName));
// invoke online configuration change
HRegionServer.configurationManager.notifyAllObservers(conf);
@@ -169,8 +186,8 @@ public class TestClassLoading {
//now load the second coprocessor too
String current = conf.get(CoprocessorHost.USER_REGION_COPROCESSOR_FROM_HDFS_KEY);
conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_FROM_HDFS_KEY,
- current + ";" + tableName + "," + pathOnHDFS2.toString() + ","
- + cpName2);
+ current + ";"
+ + CoprocessorHost.genDfsPerTableProjectEntry(CP_NAME, 2, tableName));
// invoke online config change
HRegionServer.configurationManager.notifyAllObservers(conf);
allCoprocessors = RegionCoprocessorHost.getEverLoadedCoprocessors();
@@ -179,8 +196,8 @@ public class TestClassLoading {
expectedCoprocessorSimpleName.add(cpName2);
for (HRegion r : regions) {
Set<String> currentCoprocessors = r.getCoprocessorHost().getCoprocessors();
- assertTrue("Number of currently loaded coprocessors",
- currentCoprocessors.size() == 2);
+ assertEquals("Number of currently loaded coprocessors", 2,
+ currentCoprocessors.size());
assertEquals("Expected loaded coprocessors",
expectedCoprocessorSimpleName, currentCoprocessors);
}
[07/50] [abbrv] git commit: [HBASE-9704] Fix regression from
https://phabricator.fb.com/D1363451
Posted by ec...@apache.org.
[HBASE-9704] Fix regression from https://phabricator.fb.com/D1363451
Summary: The root region location was used both by the region servers and the client to identify the communication protocol.
Test Plan: Run failed unit tests.
Reviewers: daviddeng, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1385394
Tasks: 4449946
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42901 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd018332
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd018332
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd018332
Branch: refs/heads/0.89-fb
Commit: bd0183320170bb59c7a80acaf1c33922aeeaac74
Parents: 31b1af1
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 16 22:08:36 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 6 ++
.../hadoop/hbase/client/TableServers.java | 70 ++++++++++++++------
.../org/apache/hadoop/hbase/master/HMaster.java | 3 +
.../hadoop/hbase/master/RegionManager.java | 4 +-
.../hbase/regionserver/HRegionServer.java | 10 ++-
.../hbase/zookeeper/ZooKeeperWrapper.java | 24 +++++--
6 files changed, 88 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 65f7ff8..ddaabce 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1124,6 +1124,12 @@ public final class HConstants {
public static final ProtocolVersion PROTOCOL_VERSION
= PROTOCOL_VERSION_THRIFT;
+ public static final String CLIENT_TO_RS_PROTOCOL_VERSION =
+ "hbase.client.to.rs.protocol.version";
+
+ public static final String CLIENT_TO_RS_PROTOCOL_VERSION_DEFAULT =
+ PROTOCOL_VERSION_THRIFT.name();
+
private HConstants() {
// Can't be instantiated with this constructor.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index 9c04348..a95e2ba 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -244,9 +244,7 @@ public class TableServers implements ServerConnection {
batchedUploadUpdatesMap = new ConcurrentHashMap<>();
private int batchedUploadSoftFlushRetries;
private long batchedUploadSoftFlushTimeoutMillis;
- private final boolean useThrift;
- // Zookeeper wrapper to query zookeeper for client bootstrap.
- private static ZooKeeperWrapper staticZK = null;
+ private AtomicBoolean useThrift = null;
private Map<StringBytes, StringBytes> initializedTableSet =
new ConcurrentHashMap<>();
@@ -258,21 +256,6 @@ public class TableServers implements ServerConnection {
public TableServers(Configuration conf) {
this.conf = conf;
params = HConnectionParams.getInstance(conf);
- boolean useThrift = conf.getBoolean(HConstants.CLIENT_TO_RS_USE_THRIFT,
- HConstants.CLIENT_TO_RS_USE_THRIFT_DEFAULT);
- try {
- if (staticZK == null) {
- staticZK = getZooKeeperWrapper();
- }
- RootRegionLocation loc = staticZK.readWrappedRootRegionLocation();
- if (loc != null) {
- useThrift = loc.getProtocolVersion() ==
- ProtocolVersion.THRIFT ? true : false;
- }
- } catch (Exception e) {
- LOG.error("Falling back to config based protocol version detection", e);
- }
- this.useThrift = useThrift;
String serverClassName =
conf.get(HConstants.REGION_SERVER_CLASS,
@@ -1199,6 +1182,48 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
metaCache.clear();
}
+ public static ProtocolVersion getProtocolVersionFromConf(Configuration conf) {
+ String protocolVersion = conf.get(HConstants.CLIENT_TO_RS_PROTOCOL_VERSION);
+ if (protocolVersion == null) {
+ return null;
+ }
+ try {
+ return ProtocolVersion.valueOf(protocolVersion);
+ } catch(Exception e) {
+ return null;
+ }
+ }
+
+ public void lazilySetProtocolVersionFromConfigAndRoot() {
+ if (this.useThrift != null) return;
+ synchronized (this) {
+ if (this.useThrift != null) return;
+ useThrift = new AtomicBoolean();
+ ProtocolVersion protocolVersion = getProtocolVersionFromConf(this.conf);
+ if (protocolVersion != null) {
+ if (protocolVersion == ProtocolVersion.THRIFT) {
+ useThrift.set(true);
+ } else {
+ useThrift.set(false);
+ }
+ } else {
+ try {
+ ZooKeeperWrapper staticZK = getZooKeeperWrapper();
+ RootRegionLocation loc =
+ staticZK.readWrappedRootRegionLocation(protocolVersion);
+ if (loc != null) {
+ useThrift.set(loc.getProtocolVersion() ==
+ ProtocolVersion.THRIFT ? true : false);
+ }
+ } catch (Exception e) {
+ LOG.error("Falling back to config based protocol version detection", e);
+ this.useThrift.set(conf.getBoolean(HConstants.CLIENT_TO_RS_USE_THRIFT,
+ HConstants.CLIENT_TO_RS_USE_THRIFT_DEFAULT));
+ }
+ }
+ }
+ }
+
@SuppressWarnings("resource")
@Override
public HRegionInterface getHRegionConnection(
@@ -1209,7 +1234,9 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
getMaster();
}
HRegionInterface server = HRegionServer.getMainRS(regionServer);
- if (server != null && !this.useThrift) {
+ lazilySetProtocolVersionFromConfigAndRoot();
+
+ if (server != null && !this.useThrift.get()) {
return server;
}
@@ -1222,7 +1249,7 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
// establish an RPC for this RS
// set hbase.ipc.client.connect.max.retries to retry connection
// attempts
- if (this.useThrift) {
+ if (this.useThrift.get()) {
Class<? extends ThriftClientInterface> serverInterface =
ThriftHRegionInterface.Async.class;
if (thriftPortWrittenToMeta) {
@@ -1310,7 +1337,8 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
&& localTimeouts < params.getNumRetries()) {
// Don't read root region until we're out of safe mode so we know
// that the meta regions have been assigned.
- rootRegionAddress = zk.readRootRegionLocation();
+ rootRegionAddress = zk.readRootRegionLocation(
+ getProtocolVersionFromConf(conf));
if (rootRegionAddress == null) {
try {
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0d008ff..c95f821 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.ProtocolVersion;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.StopStatus;
import org.apache.hadoop.hbase.TableExistsException;
@@ -271,6 +272,8 @@ public class HMaster extends HasThread implements HMasterInterface,
*/
public HMaster(Configuration conf) throws IOException {
this.conf = conf;
+ this.conf.set(HConstants.CLIENT_TO_RS_PROTOCOL_VERSION,
+ ProtocolVersion.THRIFT.name());
// Get my address. So what's the difference between the temp a and address?
// address.toString() will always produce an IP address as the hostname.
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java b/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
index b44d1a7..6a61715 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.TableServers;
import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -2756,7 +2757,8 @@ public class RegionManager {
/** Recovers root region location from ZK. Should only be called on master startup. */
void recoverRootRegionLocationFromZK() {
- HServerInfo rootLocationInZK = zkWrapper.readRootRegionServerInfo();
+ HServerInfo rootLocationInZK = zkWrapper.readRootRegionServerInfo(
+ TableServers.getProtocolVersionFromConf(this.master.getConfiguration()));
if (rootLocationInZK != null) {
synchronized (rootRegionLocation) {
rootRegionLocation.set(rootLocationInZK);
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5abb0ec..7cbf606 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -93,6 +93,7 @@ import org.apache.hadoop.hbase.Leases;
import org.apache.hadoop.hbase.Leases.LeaseStillHeldException;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ProtocolVersion;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.UnknownRowLockException;
import org.apache.hadoop.hbase.UnknownScannerException;
@@ -620,6 +621,9 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
this.rpcServerInfo = new HServerInfo(new HServerAddress(
new InetSocketAddress(address.getBindAddress(), port)),
System.currentTimeMillis(), machineName);
+
+ this.conf.set(HConstants.CLIENT_TO_RS_PROTOCOL_VERSION,
+ ProtocolVersion.HADOOP_RPC.name());
}
if (useThrift) {
int niftyServerPort =
@@ -643,6 +647,8 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
port = niftyThriftServer.getPort();
}
isRpcServerRunning = true;
+ this.conf.set(HConstants.CLIENT_TO_RS_PROTOCOL_VERSION,
+ ProtocolVersion.THRIFT.name());
}
this.serverInfo = new HServerInfo(new HServerAddress(
@@ -890,7 +896,9 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
for (int tries = 0; !stopRequestedAtStageOne.get() && isHealthy();) {
// Try to get the root region location from the master.
if (!haveRootRegion.get()) {
- HServerAddress rootServer = zooKeeperWrapper.readRootRegionLocation();
+ HServerAddress rootServer =
+ zooKeeperWrapper.readRootRegionLocation(
+ TableServers.getProtocolVersionFromConf(conf));
if (rootServer != null) {
// By setting the root region location, we bypass the wait imposed
// on HTable for all regions being assigned.
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd018332/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
index de7589e..6b42554 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
@@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.ProtocolVersion;
import org.apache.hadoop.hbase.RootRegionLocation;
+import org.apache.hadoop.hbase.client.TableServers;
import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
import org.apache.hadoop.hbase.util.Bytes;
@@ -183,6 +185,11 @@ public class ZooKeeperWrapper implements Watcher {
private static SimpleDateFormat format = new SimpleDateFormat("yy/MM/dd HH:mm:ss");
+ /**
+ * Configuration object to get the protocol version from.
+ */
+ private final Configuration conf;
+
private static String format(long date) {
return format.format(new Date(date));
}
@@ -259,6 +266,7 @@ public class ZooKeeperWrapper implements Watcher {
Abortable abortable) throws IOException {
this.instanceName = instanceName;
Properties properties = HQuorumPeer.makeZKProps(conf);
+ this.conf = conf;
quorumServers = HQuorumPeer.getZKQuorumServersString(properties);
if (quorumServers == null) {
throw new IOException("Could not read quorum servers from " +
@@ -394,7 +402,8 @@ public class ZooKeeperWrapper implements Watcher {
sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode);
sb.append("\n Cluster up? ").append(exists(clusterStateZNode, true));
sb.append("\n Master address: ").append(readMasterAddress(null));
- sb.append("\n Region server holding ROOT: ").append(readRootRegionLocation());
+ sb.append("\n Region server holding ROOT: ").append(readRootRegionLocation(
+ TableServers.getProtocolVersionFromConf(conf)));
sb.append("\n Region servers:");
for (HServerAddress address : scanRSDirectory()) {
sb.append("\n - ").append(address);
@@ -588,12 +597,13 @@ public class ZooKeeperWrapper implements Watcher {
* @return HServerAddress pointing to server serving root region or null if
* there was a problem reading the ZNode.
*/
- public HServerAddress readRootRegionLocation() {
- RootRegionLocation loc = readWrappedRootRegionLocation();
+ public HServerAddress readRootRegionLocation(ProtocolVersion defaultProtocolVersion) {
+ RootRegionLocation loc = readWrappedRootRegionLocation(defaultProtocolVersion);
return loc == null ? null : loc.getServerAddress();
}
- public RootRegionLocation readWrappedRootRegionLocation() {
+ public RootRegionLocation readWrappedRootRegionLocation(
+ ProtocolVersion defaultProtocolVersion) {
RootRegionLocation loc = readRootRegionLocation(rootRegionZNodeFinal, null);
if (loc != null) {
return loc;
@@ -611,8 +621,10 @@ public class ZooKeeperWrapper implements Watcher {
/**
* @return the location of the server serving the root region, including the start code
*/
- public HServerInfo readRootRegionServerInfo() {
- RootRegionLocation loc = readWrappedRootRegionLocation();
+ public HServerInfo readRootRegionServerInfo(ProtocolVersion
+ defaultProtocolVersion) {
+ RootRegionLocation loc = readWrappedRootRegionLocation(
+ defaultProtocolVersion);
if (loc == null) return null;
return new HServerInfo(loc.getServerAddress(),
loc.getStartCode(), loc.getHostName());
[34/50] [abbrv] git commit: [HBASE-9704] Fix TestHLog
Posted by ec...@apache.org.
[HBASE-9704] Fix TestHLog
Summary: Units mismatch was causing TestSlowLogRoll to fail.
Test Plan: Run and ensure that TestHLog is successfull.
Reviewers: elliott, rshroff
Reviewed By: rshroff
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1402953
Tasks: 4584691
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43149 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abbf77d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abbf77d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abbf77d9
Branch: refs/heads/0.89-fb
Commit: abbf77d9cb8458ec9753b68f46306d47e87c1961
Parents: c22acc7
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 25 17:51:47 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/abbf77d9/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 7e2036e..e7aa701 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1114,7 +1114,7 @@ public class HLog implements Syncable {
txid = this.unflushedEntries.incrementAndGet();
}
if (info.isMetaRegion()) forceSync();
- final long start = System.currentTimeMillis();
+ final long start = System.nanoTime();
return Futures.makeChecked(Futures.transform(logSyncFuture,
new AsyncFunction<Void, Long>() {
@@ -1126,7 +1126,7 @@ public class HLog implements Syncable {
writeSize.inc(len);
// Update the metrics and log down the outliers
- long end = System.currentTimeMillis();
+ long end = System.nanoTime();
long syncTime = end - start;
gsyncTime.inc(syncTime);
[24/50] [abbrv] git commit: [master] Optimize AsignmentPlan page
Posted by ec...@apache.org.
[master] Optimize AsignmentPlan page
Summary:
# Show current RS server when not in favored nodes.
# Refactoring HTML code.
Test Plan: Run TestFSUtils and connect to local http server with a browser.
Reviewers: manukranthk, elliott, adela
Reviewed By: adela
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1386043
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42985 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd54a011
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd54a011
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd54a011
Branch: refs/heads/0.89-fb
Commit: cd54a011c92a44fabe60c1bd98eede367c763bfa
Parents: a4ba645
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 19 18:40:43 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hbase-webapps/master/assignmentPlan.jsp | 130 +++++++++----------
.../resources/hbase-webapps/static/hbase.css | 16 +++
2 files changed, 80 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cd54a011/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
index 059cc28..10851bf 100644
--- a/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
+++ b/src/main/resources/hbase-webapps/master/assignmentPlan.jsp
@@ -71,82 +71,80 @@
Map<HRegionInfo, List<HServerAddress>> assignmentMap = plan.getAssignmentMap();
Map<HRegionInfo, HServerAddress> currentAssignment = snapShot.getRegionToRegionServerMap();
-%><?xml version="1.0" encoding="UTF-8" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
- <head>
- <meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <title>HBase Assignment Plan</title>
- <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
- </head>
- <body>
- <a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo_med.gif" alt="HBase Logo" title="HBase Logo" /></a>
- <h1 id="page_title">Region Assignment Plan</h1>
- <p id="links_menu"><a href="/master.jsp">Master</a></p>
- <hr id="head_rule" />
+%><!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8"/>
+ <meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+ <title>HBase Assignment Plan</title>
+ <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
+</head>
+<body>
+ <a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo_med.gif" alt="HBase Logo" title="HBase Logo" /></a>
+ <h1 id="page_title">Region Assignment Plan</h1>
+ <p id="links_menu"><a href="/master.jsp">Master</a></p>
+ <hr id="head_rule" />
- <h2>Region Assignment Plan </h2>
+ <h2>Region Assignment Plan </h2>
<%
- if (error.length() != 0) {
+ if (error.length() != 0) {
%>
- <h3>Failed to update the favored nodes: <font color="#FF0000"><%=favoredNodes%> </font>, </h3>
- <h3>for the region: <font color="#FF0000"> <%=regionName%> </font></h3>
- <h3><%=error%></h3>
+ <h3>Failed to update the favored nodes: <font color="#FF0000"><%=favoredNodes%> </font>, </h3>
+ <h3>for the region: <font color="#FF0000"> <%= StringEscapeUtils.escapeHtml(regionName) %> </font></h3>
+ <h3><%=error%></h3>
<%
- } else if (regionName != null) {
+ } else if (regionName != null) {
%>
- <h3> Succeeded to update the favored nodes: <font color="#32CD32"><%=favoredNodes%> </font>,</h3>
- <h3> for the region: <font color="#32CD32"><%=regionName%> </font></h3>
+ <h3> Succeeded to update the favored nodes: <font color="#32CD32"><%=favoredNodes%> </font>,</h3>
+ <h3> for the region: <font color="#32CD32"><%= StringEscapeUtils.escapeHtml(regionName) %> </font></h3>
<%
- }
+ }
%>
- <table>
- <tr>
- <th>Region Name</th> <th>Position</th> <th>Primary RS</th> <th>Secondary RS</th> <th>Tertiary RS</th> <th>Actions</th>
- </tr>
+ <table>
+ <tr>
+ <th>Region Name</th>
+ <th>Position</th>
+ <th>Primary RS</th>
+ <th>Secondary RS</th>
+ <th>Tertiary RS</th>
+ <th>Actions</th>
+ </tr>
<%
- for (Map.Entry<String, HRegionInfo> entry : regionNameToRegionInfoMap.entrySet()) {
- HRegionInfo regionInfo = entry.getValue();
- List<HServerAddress> favoredNodeList = assignmentMap.get(regionInfo);
- regionName = regionInfo.getRegionNameAsString();
- HServerAddress currentRS = currentAssignment.get(regionInfo);
- String position = "Not Assigned";
- AssignmentPlan.POSITION favoredNodePosition =
- AssignmentPlan.getFavoredServerPosition(favoredNodeList, currentRS);
- if (favoredNodePosition == null) {
- position = "Not on FavoredNode";
- } else {
- position = favoredNodePosition.toString();
- }
+ for (Map.Entry<String, HRegionInfo> entry : regionNameToRegionInfoMap.entrySet()) {
+ HRegionInfo regionInfo = entry.getValue();
+ List<HServerAddress> favoredNodeList = assignmentMap.get(regionInfo);
+ regionName = regionInfo.getRegionNameAsString();
+
+ String position = "Not Assigned";
+ String tdClass = "bad";
+
+ HServerAddress currentRS = currentAssignment.get(regionInfo);
+ if (currentRS != null) {
+ AssignmentPlan.POSITION favoredNodePosition =
+ AssignmentPlan.getFavoredServerPosition(favoredNodeList, currentRS);
+ if (favoredNodePosition == null) {
+ position = "" + currentRS;
+ } else {
+ position = favoredNodePosition.toString();
+ if (favoredNodePosition == AssignmentPlan.POSITION.PRIMARY) {
+ tdClass = "good";
+ } else {
+ tdClass = "warn";
+ }
+ }
+ }
%>
- <tr>
- <form method="post">
- <input type="hidden" name="regionName" value="<%= StringEscapeUtils.escapeHtml(regionName) %>">
- <td><%= StringEscapeUtils.escapeHtml(regionName) %> </td>
- <%
- if (position.startsWith("Not")) {
- %>
- <td><b><font color="#FF0000"><%=position%></font></b></td>
- <%
- } else if (position.equalsIgnoreCase(AssignmentPlan.POSITION.PRIMARY.toString())){
- %>
- <td><b><font color="#32CD32"><%=position%></font></b></td>
- <%
- } else {
- %>
- <td><b><font color="#FFD700"><%=position%></font></b></td>
- <%
- }
- %>
- <td><input type="text" size="40" name="primaryRS" value="<%=favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.PRIMARY.ordinal()).getHostNameWithPort()%>"</td>
- <td><input type="text" size="40" name="secondaryRS" value="<%=favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.SECONDARY.ordinal()).getHostNameWithPort()%>"</td>
- <td><input type="text" size="40" name="tertiaryRS" value="<%=favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.TERTIARY.ordinal()).getHostNameWithPort()%>"</td>
- <td><input type="submit" size="5" value="Update"></td>
- </form>
- </tr>
+ <tr><form method="post">
+ <input type="hidden" name="regionName" value="<%= StringEscapeUtils.escapeHtml(regionName) %>">
+ <td><%= StringEscapeUtils.escapeHtml(regionName) %></td>
+ <td class="<%= tdClass %>"><%= position %></td>
+ <td><input type="text" size="40" name="primaryRS" value="<%= favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.PRIMARY.ordinal()).getHostNameWithPort() %>"</td>
+ <td><input type="text" size="40" name="secondaryRS" value="<%= favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.SECONDARY.ordinal()).getHostNameWithPort() %>"</td>
+ <td><input type="text" size="40" name="tertiaryRS" value="<%= favoredNodeList == null ? "" : favoredNodeList.get(AssignmentPlan.POSITION.TERTIARY.ordinal()).getHostNameWithPort() %>"</td>
+ <td><input type="submit" size="5" value="Update"></td>
+ </form></tr>
<%
- }
+ }
%>
</table>
</body>
http://git-wip-us.apache.org/repos/asf/hbase/blob/cd54a011/src/main/resources/hbase-webapps/static/hbase.css
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/static/hbase.css b/src/main/resources/hbase-webapps/static/hbase.css
index 8f2063a..d659e0f 100644
--- a/src/main/resources/hbase-webapps/static/hbase.css
+++ b/src/main/resources/hbase-webapps/static/hbase.css
@@ -35,3 +35,19 @@ tr.task-monitor-ABORTED td {
td.number {
text-align: right;
}
+
+td.bad, td.warn, td.good {
+ font-weight: bold;
+}
+
+td.good {
+ color: #32CD32;
+}
+
+td.warn {
+ color: #FFD700;
+}
+
+td.bad {
+ color: red;
+}
[45/50] [abbrv] git commit: [master] Increase idle time on a
connection to one day
Posted by ec...@apache.org.
[master] Increase idle time on a connection to one day
Test Plan: not tested
Reviewers: manukranthk, daviddeng, gauravm, fan
Reviewed By: fan
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1425597
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43429 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fede35dd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fede35dd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fede35dd
Branch: refs/heads/0.89-fb
Commit: fede35dd59efe3926f884170d1996adcd6248be6
Parents: 8e1133c
Author: adela <ad...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jul 10 21:54:21 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
src/main/java/org/apache/hadoop/hbase/HConstants.java | 6 ++++++
.../org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java | 6 ++++--
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fede35dd/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3970bc2..8be87a4 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -212,6 +212,12 @@ public final class HConstants {
/** Default value of REGIONSERVER_USE_HADOOP_RPC boolean flag **/
public static final boolean DEFAULT_REGIONSERVER_USE_HADOOP_RPC = false;
+ /** String representing the name of the config for maximal idle time of a connection **/
+ public static final String SWIFT_CONNECTION_IDLE_MAX_MINUTES = "swift.connection.idle.max";
+
+ /** Default value of SWIFT_CONNECTION_IDLE_MAX_MINUTES*/
+ public static final int SWIFT_CONNECTION_IDLE_MAX_MINUTES_DEFAULT = 10;
+
/** Boolean flag to represent whether to use THRIFT or not **/
public static final String REGIONSERVER_USE_THRIFT = "hbase.regionserver.rpc.thrift.enabled";
http://git-wip-us.apache.org/repos/asf/hbase/blob/fede35dd/src/main/java/org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java
index 4a6123a..4ba0486 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/HBaseNiftyThriftServer.java
@@ -111,8 +111,10 @@ public class HBaseNiftyThriftServer {
conf.getInt(HConstants.SWIFT_WORKER_THREADS,
HConstants.SWIFT_WORKER_THREADS_DEFAULT))
.setIdleConnectionTimeout(
- new Duration(conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
- HConstants.DEFAULT_HBASE_RPC_TIMEOUT), TimeUnit.MILLISECONDS));
+ new Duration(conf.getInt(
+ HConstants.SWIFT_CONNECTION_IDLE_MAX_MINUTES,
+ HConstants.SWIFT_CONNECTION_IDLE_MAX_MINUTES_DEFAULT),
+ TimeUnit.MINUTES));
if (useHeaderProtocol) {
server = new ThriftServer(processor, serverConfig.setProtocolName(
[10/50] [abbrv] git commit: [HBASE-11303] New feature: expand cluster
on specific hosts
Posted by ec...@apache.org.
[HBASE-11303] New feature: expand cluster on specific hosts
Summary:
we need to be able to expand a cluster onto specified hosts. Also this is a better solution then the expandRack, since the moving is done on per table basis.
- We first try to guess the new regionservers - but user can still specify them if we didn't guess correctly.
- Then we keep a priorityqueue of regionservers (sorted in descending order) by number of regions assigned.
- We take regions from these servers and move the tertiary to a new location
- RegionPlacement -u should be run in the end to complete the movement
Test Plan: will write a unit test
Reviewers: manukranthk, liyintang, aaiyer, fan, gauravm, rshroff, daviddeng
Reviewed By: daviddeng
Subscribers: elliott, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1332359
Tasks: 3041849
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42718 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/91eea3f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/91eea3f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/91eea3f6
Branch: refs/heads/0.89-fb
Commit: 91eea3f69f6ff07e493b0acaf2a256c2122c1ecd
Parents: ce2595d
Author: adela <ad...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jun 6 17:08:07 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/AssignmentDomain.java | 31 ++-
.../hbase/master/RegionAssignmentSnapshot.java | 26 ++-
.../hadoop/hbase/master/RegionPlacement.java | 226 +++++++++++++++----
.../hbase/master/RegionPlacementTestBase.java | 35 ++-
.../hbase/master/TestRegionPlacement.java | 129 ++++++++++-
5 files changed, 370 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/91eea3f6/src/main/java/org/apache/hadoop/hbase/master/AssignmentDomain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentDomain.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentDomain.java
index 29e97ea..5d10a53 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentDomain.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentDomain.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
-import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +37,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
+import com.google.common.collect.Lists;
+
public class AssignmentDomain {
protected static final Log LOG =
LogFactory.getLog(AssignmentDomain.class.getClass());
@@ -231,4 +232,32 @@ public class AssignmentDomain {
return false;
return true;
}
+
+ /**
+ * Given list of hostnames (Set of strings) you will get list of
+ * HServerAddress. The order will be perserved i.e if you passed in the set
+ * (h1, 2, 3) you will get (HServerAddres1, 2, 3)
+ *
+ * @param hostnames
+ * - Set of String
+ * @return - List of corresponding HServerAddress
+ */
+ public List<HServerAddress> getHServerAddressFromHostname(
+ Set<String> hostnames) {
+ List<HServerAddress> serversToReturn = new ArrayList<HServerAddress>();
+ Set<HServerAddress> servers = regionServerToRackMap.keySet();
+ Map<String, HServerAddress> allServerNameToAddress = new HashMap<>();
+ for (HServerAddress address : servers) {
+ allServerNameToAddress.put(address.getHostname(), address);
+ }
+ for (String host : hostnames) {
+ HServerAddress hostAddress = allServerNameToAddress.get(host);
+ if (hostAddress == null) {
+ LOG.error("Host: " + hostAddress + " is not recognized!!!");
+ } else {
+ serversToReturn.add(hostAddress);
+ }
+ }
+ return serversToReturn;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91eea3f6/src/main/java/org/apache/hadoop/hbase/master/RegionAssignmentSnapshot.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionAssignmentSnapshot.java b/src/main/java/org/apache/hadoop/hbase/master/RegionAssignmentSnapshot.java
index 9e80713..75b9df1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionAssignmentSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/RegionAssignmentSnapshot.java
@@ -54,6 +54,7 @@ public class RegionAssignmentSnapshot {
/** the region name to region info map */
private final Map<String, HRegionInfo> regionNameToRegionInfoMap;
+ private final Map<String, Map<HServerAddress, List<HRegionInfo>>> rsToRegionsPerTable;
/** the regionServer to region map */
private final Map<HServerAddress, List<HRegionInfo>> regionServerToRegionMap;
/** the existing assignment plan in the META region */
@@ -67,6 +68,7 @@ public class RegionAssignmentSnapshot {
regionToRegionServerMap = new HashMap<HRegionInfo, HServerAddress>();
regionServerToRegionMap = new HashMap<HServerAddress, List<HRegionInfo>>();
regionNameToRegionInfoMap = new TreeMap<String, HRegionInfo>();
+ rsToRegionsPerTable = new HashMap<>();
exsitingAssignmentPlan = new AssignmentPlan();
globalAssignmentDomain = new AssignmentDomain(conf);
}
@@ -77,7 +79,7 @@ public class RegionAssignmentSnapshot {
*/
public void initialize() throws IOException {
LOG.info("Start to scan the META for the current region assignment " +
- "snappshot");
+ "snapshot");
// Add all the online region servers
HBaseAdmin admin = new HBaseAdmin(conf);
@@ -160,6 +162,20 @@ public class RegionAssignmentSnapshot {
}
regionList.add(regionInfo);
regionServerToRegionMap.put(server, regionList);
+
+ // update rsToRegionsPerTable accordingly
+ String tblName = regionInfo.getTableDesc().getNameAsString();
+ Map<HServerAddress, List<HRegionInfo>> assignment = rsToRegionsPerTable.get(tblName);
+ if (assignment== null){
+ assignment = new HashMap<>();
+ rsToRegionsPerTable.put(tblName, assignment);
+ }
+ List<HRegionInfo> regions = assignment.get(server);
+ if (regions == null) {
+ regions = new ArrayList<>();
+ assignment.put(server, regions);
+ }
+ regions.add(regionInfo);
}
}
@@ -190,4 +206,12 @@ public class RegionAssignmentSnapshot {
public Set<String> getTableSet() {
return this.tableToRegionMap.keySet();
}
+
+ /**
+ * Returns assignment regionserver->regions per table
+ *
+ */
+ public Map<String, Map<HServerAddress, List<HRegionInfo>>> getRegionServerToRegionsPerTable() {
+ return this.rsToRegionsPerTable;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91eea3f6/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java b/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
index b91c29e..6c7e7aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.PriorityQueue;
import java.util.Random;
import java.util.Scanner;
import java.util.Set;
@@ -640,6 +641,107 @@ public class RegionPlacement implements RegionPlacementPolicy{
}
/**
+ * Used to expand cluster by adding more hosts. You don't need to use this
+ * when you are adding just a few hosts, but when you add a bunch of them
+ * (let's say >5 at once)
+ *
+ * We are iterating through each table and we are moving one region at a time
+ * starting from the hosts which contain largest number of regions. Note that
+ * only the tertiary will be updated, you will still need to update the
+ * assignment plan after a few days after locality builds up.
+ *
+ * @param newHosts
+ * - new hosts where we expand the cluster
+ * @param assignmentSnapshot
+ * - current assignment snapshot
+ * @throws IOException
+ */
+ public AssignmentPlan expandRegionsToNewHosts(List<HServerAddress> newHosts,
+ RegionAssignmentSnapshot assignmentSnapshot) throws IOException {
+
+ AssignmentPlan plan = assignmentSnapshot.getExistingAssignmentPlan();
+
+ Set<String> allTables = assignmentSnapshot.getTableSet();
+ for (String t : allTables) {
+ int totalRegionsPerTable = 0;
+ PriorityQueue<Pair<HServerAddress, List<HRegionInfo>>> pqueue = new PriorityQueue<>(
+ 200, new Comparator<Pair<HServerAddress, List<HRegionInfo>>>() {
+
+ @Override
+ public int compare(Pair<HServerAddress, List<HRegionInfo>> p1,
+ Pair<HServerAddress, List<HRegionInfo>> p2) {
+ return -Integer.compare(p1.getSecond().size(), p2.getSecond()
+ .size());
+ }
+ });
+ Map<HServerAddress, List<HRegionInfo>> rsToRegionsPerTable = assignmentSnapshot
+ .getRegionServerToRegionsPerTable().get(t);
+ for (Entry<HServerAddress, List<HRegionInfo>> entry : rsToRegionsPerTable.entrySet()) {
+ System.out.println("server: " + entry.getKey() + " regions" + entry.getValue().size());
+ totalRegionsPerTable +=entry.getValue().size();
+ }
+ for (Entry<HServerAddress, List<HRegionInfo>> entry : rsToRegionsPerTable
+ .entrySet()) {
+ pqueue.add(new Pair<HServerAddress, List<HRegionInfo>>(entry.getKey(),
+ entry.getValue()));
+ }
+
+ // for each of the new machines - calculate how many regions of this table
+ // should be placed
+
+ // calculate overall avg of regions per host (including new hosts as well)
+ AssignmentDomain domain = assignmentSnapshot.getGlobalAssignmentDomain();
+ System.out.println("rs: " + domain.getAllServers().size());
+ double avgPerHosts = (double) totalRegionsPerTable
+ / domain.getAllServers().size();
+ int avgPerHostFloor = (int) Math.floor(avgPerHosts);
+ Random rand = new Random();
+ for (HServerAddress server : newHosts) {
+ int neededRegionsOnNewHost = avgPerHostFloor
+ + (rand.nextDouble() < (avgPerHosts - avgPerHostFloor) ? 1 : 0);
+ // now start taking regions from the existing hosts (they are sorted by
+ // number of regions in descending order)
+ int placedRegionOnNew = 0;
+ while (placedRegionOnNew < neededRegionsOnNewHost) {
+ Pair<HServerAddress, List<HRegionInfo>> onOld = pqueue.poll();
+ HRegionInfo regionToMove = onOld.getSecond().remove(0);
+ // now return back the modified pair in the priority queue
+ pqueue.add(onOld);
+ placedRegionOnNew++;
+ // move the tertiary of the region to the new server
+ List<HServerAddress> favoredNodes = plan.getAssignment(regionToMove);
+ AssignmentPlan.replaceFavoredNodesServerWithNew(favoredNodes,
+ AssignmentPlan.POSITION.TERTIARY, server);
+ plan.updateAssignmentPlan(regionToMove, favoredNodes);
+ }
+ System.out.println("Server: " + server + " will get "
+ + placedRegionOnNew + " tertiary assignments for table " + t);
+ }
+ }
+ return plan;
+ }
+
+ /**
+ * @param plan
+ * @throws IOException
+ */
+ private void userUpdatePlan(AssignmentPlan plan) throws IOException {
+ System.out
+ .println("Do you want to update the assignment plan with this changes (y/n): ");
+ Scanner s = new Scanner(System.in);
+ String input = s.nextLine().trim();
+ s.close();
+ if (input.toLowerCase().equals("y")) {
+ System.out.println("Updating assignment plan...");
+ updateAssignmentPlanToMeta(plan);
+ updateAssignmentPlanToRegionServers(plan);
+ } else {
+ System.out.println("exiting without updating the assignment plan");
+ }
+ }
+
+
+ /**
* This method will pick regions from a given rack, such that these regions
* are going to be moved to the new rack later. The logic behind is: we move
* the regions' tertiaries into a new rack
@@ -702,12 +804,14 @@ public class RegionPlacement implements RegionPlacementPolicy{
serverIndex++;
}
System.out.println("Total number of regions: " + totalMovedPerRack
- + " in rack " + currentRack + " will move its tertiary to a new rack: "
+ + " in rack " + currentRack + " will move its tertiary to the new : "
+ newRack);
System.out.println("------------------------------------------");
return regionsToMove;
}
+
+
/**
* Returns the average number of regions per regionserver
*
@@ -730,11 +834,15 @@ public class RegionPlacement implements RegionPlacementPolicy{
* Move the regions to the new rack, such that each server will get equal
* number of regions
*
- * @param plan - the current assignment plan
- * @param domain - the assignment domain
- * @param regionsToMove - the regions that would be moved to the new rack
- * regionserver per rack are picked to be moved in the new rack
- * @param newRack - the new rack
+ * @param plan
+ * - the current assignment plan
+ * @param domain
+ * - the assignment domain
+ * @param regionsToMove
+ * - the regions that would be moved to the new rack regionserver per
+ * rack are picked to be moved in the new rack
+ * @param newRack
+ * - the new rack
* @throws IOException
*/
private void moveRegionsToNewRack(AssignmentPlan plan,
@@ -742,7 +850,8 @@ public class RegionPlacement implements RegionPlacementPolicy{
throws IOException {
System.out.println("------------------------------------------");
System.out
- .println("Printing how many regions are planned to be assigned per region server in the new rack (" + newRack + ")");
+ .println("Printing how many regions are planned to be assigned per region server in the new rack ("
+ + newRack + ")");
List<HServerAddress> serversFromNewRack = domain
.getServersFromRack(newRack);
int totalNumRSNewRack = serversFromNewRack.size();
@@ -761,19 +870,9 @@ public class RegionPlacement implements RegionPlacementPolicy{
System.out.println("RS: " + serversFromNewRack.get(j).getHostname()
+ " got " + regionsPerRs + "tertiary regions");
}
- System.out
- .println("Do you want to update the assignment plan with this changes (y/n): ");
- Scanner s = new Scanner(System.in);
- String input = s.nextLine().trim();
- s.close();
- if (input.toLowerCase().equals("y")) {
- System.out.println("Updating assignment plan...");
- updateAssignmentPlanToMeta(plan);
- updateAssignmentPlanToRegionServers(plan);
- } else {
- System.out.println("exiting without updating the assignment plan");
- }
+ userUpdatePlan(plan);
}
+
/**
* Generate the assignment plan for the existing table
*
@@ -1657,6 +1756,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
"use munkres to place secondaries and tertiaries");
opt.addOption("ld", "locality-dispersion", false, "print locality and dispersion information for current plan");
opt.addOption("exprack", "expand-with-rack", false, "expand the regions to a new rack");
+ opt.addOption("expHosts", false, "expand the regions to a new rack");
opt.addOption("rnum", false, "print number of primaries per RS");
opt.addOption("bp", "balance-primary", false, "balance the primaries across all regionservers in the cluster");
try {
@@ -1733,14 +1833,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
.getRegionDegreeLocalityMappingFromFS(conf);
Map<String, Integer> movesPerTable = rp.getRegionsMovement(newPlan);
rp.checkDifferencesWithOldPlan(movesPerTable, locality, newPlan);
- System.out.println("Do you want to update the assignment plan? [y/n]");
- Scanner s = new Scanner(System.in);
- String input = s.nextLine().trim();
- if (input.equals("y")) {
- System.out.println("Updating assignment plan...");
- rp.updateAssignmentPlan(newPlan);
- }
- s.close();
+ rp.userUpdatePlan(newPlan);
}
// Read all the modes
else if (cmd.hasOption("v") || cmd.hasOption("verify")) {
@@ -1773,14 +1866,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
System.out.println("Printing how will distribution of primaries look like");
rp.printDistributionOfPrimariesPerTable(rp.getRegionAssignmentSnapshot(), newPlan);
rp.printDistributionOfPrimariesPerCell(rp.getExistingAssignmentPlan(), newPlan);
- System.out.println("Do you want to update the assignment plan? [y/n]");
- Scanner s = new Scanner(System.in);
- String input = s.nextLine().trim();
- if (input.equals("y")) {
- System.out.println("Updating assignment plan...");
- rp.updateAssignmentPlan(newPlan);
- }
- s.close();
+ rp.userUpdatePlan(newPlan);
} else if (cmd.hasOption("ld")) {
Map<String, Map<String, Float>> locality = FSUtils
.getRegionDegreeLocalityMappingFromFS(conf);
@@ -1831,6 +1917,8 @@ public class RegionPlacement implements RegionPlacementPolicy{
String newRack = s.nextLine().trim();
s.close();
rp.expandRegionsToNewRack(newRack, snapshot);
+ } else if (cmd.hasOption("expHosts")) {
+ expandClusterWithNewHosts(rp);
} else if (cmd.hasOption("upload")) {
String fileName = cmd.getOptionValue("upload");
try {
@@ -1840,14 +1928,7 @@ public class RegionPlacement implements RegionPlacementPolicy{
.getRegionDegreeLocalityMappingFromFS(conf);
Map<String, Integer> movesPerTable = rp.getRegionsMovement(newPlan);
rp.checkDifferencesWithOldPlan(movesPerTable, locality, newPlan);
- System.out.println("Do you want to update the assignment plan? [y/n]");
- Scanner s = new Scanner(System.in);
- String input = s.nextLine().trim();
- if (input.equals("y")) {
- System.out.println("Updating assignment plan...");
- rp.updateAssignmentPlan(newPlan);
- }
- s.close();
+ rp.userUpdatePlan(newPlan);
} catch (JsonParseException je) {
LOG.error("Unable to parse json file", je);
} catch (IOException e) {
@@ -1874,6 +1955,65 @@ public class RegionPlacement implements RegionPlacementPolicy{
}
}
+ /**
+ * Entry-point when expanding a cluster with a number of hosts
+ *
+ * @param rp
+ * - regionPlacement object
+ * @throws IOException
+ */
+ private static void expandClusterWithNewHosts(RegionPlacement rp)
+ throws IOException {
+ RegionAssignmentSnapshot snapshot = rp.getRegionAssignmentSnapshot();
+ System.out.println("List of all hosts with #regions assigned: ");
+ Map<HServerAddress, List<HRegionInfo>> serverToRegions = snapshot
+ .getRegionServerToRegionMap();
+ // these are potential hosts - to which we want to expand
+ List<HServerAddress> emptyHosts = new ArrayList<>();
+ for (Entry<HServerAddress, List<HRegionInfo>> entry : serverToRegions
+ .entrySet()) {
+ System.out.println(entry.getKey().getHostname() + "\t:\t"
+ + entry.getValue().size());
+ if (entry.getValue().size() == 0) {
+ emptyHosts.add(entry.getKey());
+ }
+ }
+ System.out
+ .println("Guessing with which hosts you want to expand the cluster");
+ StringBuilder sb = new StringBuilder();
+ for (HServerAddress server : emptyHosts) {
+ sb.append(server.getHostname());
+ sb.append(",");
+ }
+ // remove the last comma
+ sb.deleteCharAt(sb.length() - 1);
+ System.out.println("If all the hosts are correct just press Y, otherwise "
+ + "specify the list of the hosts in one line (comma separated)");
+ System.out.println("Example:");
+ System.out.println();
+ System.out.println("hbase123.xyz, hbase456.xyz, hbase789.xyz");
+ Scanner s = new Scanner(System.in);
+ String answer = s.nextLine().trim();
+ s.close();
+ List<HServerAddress> specifiedHosts = null;
+ if (!answer.equalsIgnoreCase("Y")) {
+ Set<String> specifiedStrings = new HashSet<String>();
+ String[] hosts = answer.split(",");
+ for (String host : hosts) {
+ specifiedStrings.add(host.trim());
+ }
+ specifiedHosts = snapshot.getGlobalAssignmentDomain()
+ .getHServerAddressFromHostname(specifiedStrings);
+ if (specifiedHosts == null) {
+ System.out.println("One or more of the specified hosts were not recognized, ABORTING");
+ return;
+ }
+ }
+ AssignmentPlan newPlan = rp.expandRegionsToNewHosts(
+ specifiedHosts == null ? emptyHosts : specifiedHosts, snapshot);
+ rp.userUpdatePlan(newPlan);
+ }
+
private static void printHelp(Options opt) {
new HelpFormatter().printHelp(
"RegionPlacement < -w | -u | -n | -v | -t | -h | -overwrite -r regionName -f favoredNodes " +
http://git-wip-us.apache.org/repos/asf/hbase/blob/91eea3f6/src/test/java/org/apache/hadoop/hbase/master/RegionPlacementTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/RegionPlacementTestBase.java b/src/test/java/org/apache/hadoop/hbase/master/RegionPlacementTestBase.java
index 873a33f..7ef600d 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/RegionPlacementTestBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/RegionPlacementTestBase.java
@@ -19,41 +19,34 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
public class RegionPlacementTestBase {
protected final static RegionMovementTestHelper TEST_UTIL = new RegionMovementTestHelper();
protected final static int META_REGION_OVERHEAD = 1;
http://git-wip-us.apache.org/repos/asf/hbase/blob/91eea3f6/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
index 60cea48..83ac5bf 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
@@ -19,7 +19,21 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
@@ -32,21 +46,11 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
@Category(LargeTests.class)
public class TestRegionPlacement extends RegionPlacementTestBase {
@@ -339,5 +343,108 @@ public class TestRegionPlacement extends RegionPlacementTestBase {
assertEquals("Loaded plan should be the same with current plan", currentPlan, loadedPlan);
}
+
+ /**
+ * First create a table on pinned servers. Then change the table descriptor
+ * such that the table is not pinned anymore and can be assigned on all
+ * machines in the cluster. Then we call expandOnHosts passing the empty
+ * regionserver and in the end we verify that he received avg number of
+ * tertiaries
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testExpandOnHosts() throws IOException, InterruptedException {
+ String tableName = "testExpandHosts";
+ // let's have more region for this test case
+ REGION_NUM = 20;
+
+ TEST_UTIL.resetLastOpenedRegionCount();
+ resetLastRegionOnPrimary();
+
+ try {
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor("d"));
+
+ assertTrue("number of slaves is smaller then 3", SLAVES >= 3);
+ Set<HServerAddress> servers = new HashSet<>(3);
+ HRegionServer unusedServer = cluster.getRegionServer(3);
+
+ for (int i = 0; i < 3; i++) {
+ servers.add(cluster.getRegionServer(i).getServerInfo()
+ .getServerAddress());
+ }
+
+ htd.setServers(servers);
+ admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ admin.createTable(htd, Bytes.toBytes("aaaa"), Bytes.toBytes("zzzz"),
+ REGION_NUM);
+
+ // Wait for things to stabilize
+ TEST_UTIL.waitOnTable(tableName);
+ TEST_UTIL.waitOnStableRegionMovement();
+
+ // Reset all of the counters.
+ resetLastRegionOnPrimary();
+ TEST_UTIL.resetLastOpenedRegionCount();
+
+ verifyRegionAssignment(rp.getExistingAssignmentPlan(), 0, REGION_NUM);
+ assertPinned(tableName, cluster, servers, unusedServer);
+
+ // change the table descriptor now
+ htd.setServers(null);
+ admin.disableTable(tableName);
+ admin.modifyTable(Bytes.toBytes(tableName), htd);
+ admin.enableTable(tableName);
+
+ // Wait for things to stabilize
+ TEST_UTIL.waitOnTable(tableName);
+ TEST_UTIL.waitOnStableRegionMovement();
+
+ // Reset all of the counters.
+ resetLastRegionOnPrimary();
+ TEST_UTIL.resetLastOpenedRegionCount();
+
+ //verify there are no primaries on the last hosts from the existing table
+ HServerAddress newHost = cluster.getRegionServer(3).getServerInfo()
+ .getServerAddress();
+ Map<HRegionInfo, List<HServerAddress>> currentMap = rp.getExistingAssignmentPlan().getAssignmentMap();
+ for (List<HServerAddress> val : currentMap.values()) {
+ if (val.contains(newHost)) {
+ Assert.fail("new regionserver is contained in existing assignment");
+ }
+ }
+
+ List<HServerAddress> newHosts = new ArrayList<>();
+ newHosts.add(newHost);
+ AssignmentPlan newPlan = rp.expandRegionsToNewHosts(newHosts, rp.getRegionAssignmentSnapshot());
+ rp.updateAssignmentPlan(newPlan);
+
+ // verify there are tertiaries on the new host
+ Map<HRegionInfo, List<HServerAddress>> assignMap = newPlan.getAssignmentMap();
+ int regionsOnNewHost = 0;
+ for (List<HServerAddress> serversFromAssignment : assignMap.values()) {
+ //verify it is the tertiary
+ if (serversFromAssignment.get(AssignmentPlan.POSITION.TERTIARY.ordinal()).equals(newHost)) {
+ regionsOnNewHost++;
+ }
+ }
+ // we expect an average number of regions to move to the new host
+ assertEquals("avg number of regions to the new host: ", (int) REGION_NUM
+ / SLAVES, regionsOnNewHost);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ } finally {
+ if (admin != null) {
+ admin.disableTable(tableName);
+ admin.deleteTable(tableName);
+ admin.close();
+ admin = null;
+ }
+ }
+ }
+
}
[40/50] [abbrv] git commit: [HBASE-11379] fixed exception-handling in
the writing of .regioninfo files
Posted by ec...@apache.org.
[HBASE-11379] fixed exception-handling in the writing of .regioninfo files
Summary: error exceptions when writing .regioninfo files were not being handled properly
Test Plan: n/a
Reviewers: elliott, daviddeng
Reviewed By: daviddeng
Subscribers: elliott, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1418950
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43315 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/451e39bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/451e39bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/451e39bf
Branch: refs/heads/0.89-fb
Commit: 451e39bf9c8c6820054ddb9c9be7830be5b9847b
Parents: 797de59
Author: yuq <yu...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jul 4 00:39:41 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 41 ++++++++++----------
.../org/apache/hadoop/hbase/master/HMaster.java | 32 +++++++--------
.../hadoop/hbase/regionserver/HRegion.java | 4 +-
.../client/TestHBaseFsckFixRegionInfo.java | 4 --
4 files changed, 36 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/451e39bf/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 63701cb..54832ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -708,13 +708,12 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
* This method writes the .regioninfo file. This method is synchronized so that no two threads
* attempt to write the same .regioninfo file at the same time.
* @param conf
- * @return true if the write was successful
* @throws IOException
*/
- public synchronized boolean writeToDisk(Configuration conf) throws IOException {
+ public synchronized void writeToDisk(Configuration conf) throws IOException {
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
- return writeToDisk(conf, fs);
+ writeToDisk(conf, fs);
}
/**
@@ -722,34 +721,34 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
* attempt to write the same .regioninfo file at the same time.
* @param conf
* @param fs
- * @return true if the write is successful
* @throws IOException
*/
- public synchronized boolean writeToDisk(Configuration conf, FileSystem fs) throws IOException {
+ public synchronized void writeToDisk(Configuration conf, FileSystem fs) throws IOException {
Path tableDir =
HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), this.getTableDesc().getName());
Path regionPath = HRegion.getRegionDir(tableDir, this.getEncodedName());
Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
- FSDataOutputStream out = fs.create(regionInfoPath, true);
- boolean successfulWrite = true;
try {
- // we write to file the information necessary to reconstruct this HRegionInfo via readFields()
- // this.toString() calls tableDesc.toString(), so this file will contain back-up information
- // for the table
- write(out);
- out.write(NEWLINE);
- out.write(NEWLINE);
- out.write(Bytes.toBytes(this.toString()));
- LOG.debug("Rewrote .regioninfo file of " + this.getRegionNameAsString() + " at "
- + regionInfoPath);
+ FSDataOutputStream out = fs.create(regionInfoPath);
+ try {
+ // we write to file the information necessary to reconstruct this HRegionInfo via
+ // readFields()
+ // this.toString() calls tableDesc.toString(), so this file will contain back-up information
+ // for the table
+ write(out);
+ out.write(NEWLINE);
+ out.write(NEWLINE);
+ out.write(Bytes.toBytes(this.toString()));
+ LOG.debug("Rewrote .regioninfo file of " + this.getRegionNameAsString() + " at "
+ + regionInfoPath);
+ } finally {
+ out.close();
+ LOG.debug(".regioninfo File Contents: " + this.toString());
+ }
} catch (IOException e) {
- successfulWrite = false;
LOG.error("Could not rewrite the .regioninfo of " + this.getRegionNameAsString() + " at "
+ regionInfoPath, e);
- } finally {
- out.close();
- LOG.debug(".regioninfo File Contents: " + this.toString());
+ throw e;
}
- return successfulWrite;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/451e39bf/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9b51280..959a070 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -42,6 +42,7 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -112,6 +113,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ExceptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.InfoServer;
@@ -1670,38 +1672,32 @@ HMasterRegionInterface, Watcher, StoppableMaster {
* This method rewrites all .regioninfo files for all regions belonging to the Table whose name is
* tableName
* @param tableName
- * @return true if all .regioninfo files that belong to the table are successfully rewritten.
* @throws IOException
* @throws ExecutionException
* @throws InterruptedException
*/
- public boolean writeRegionInfo(byte[] tableName) throws Exception {
+ public void writeRegionInfo(byte[] tableName) throws IOException, InterruptedException{
List<Pair<HRegionInfo, HServerAddress>> tableRegions = getTableRegions(tableName);
ExecutorService executor = Executors.newCachedThreadPool();
- List<Future<Boolean>> futures = new ArrayList<>();
+ List<Future<Void>> futures = new ArrayList<>();
for (final Pair<HRegionInfo, HServerAddress> entry : tableRegions) {
- Callable<Boolean> writer = new Callable<Boolean>() {
+ Callable<Void> writer = new Callable<Void>() {
- // return true if the write is successful
- public Boolean call() {
+ public Void call() throws IOException {
HRegionInfo hri = entry.getFirst();
- try {
- return hri.writeToDisk(conf);
- } catch (IOException e) {
- LOG.error("Failed to write .regioninfo for Region: " + hri.getRegionNameAsString(), e);
- return false;
- }
+ hri.writeToDisk(conf);
+ return null;
}
};
- Future<Boolean> future = executor.submit(writer);
- futures.add(future);
+ futures.add(executor.submit(writer));
}
- for (Future<Boolean> f : futures) {
- if (!f.get()) { // if any write was not successful
- return false;
+ for (Future<Void> f : futures) {
+ try {
+ f.get();
+ } catch (ExecutionException e) {
+ throw ExceptionUtils.toIOException(e.getCause());
}
}
- return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/451e39bf/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index de8ba0b..4f7b42c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -617,7 +617,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
try {
// Write HRI to a file in case we need to recover .META.
status.setStatus("Writing region info on filesystem");
- checkRegioninfoOnFilesystem();
+ writeRegioninfoOnFilesystem();
// Remove temporary data left over from old regions
status.setStatus("Cleaning up temporary data from old regions");
@@ -756,7 +756,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* mangled regions.
* @throws IOException
*/
- private void checkRegioninfoOnFilesystem() throws IOException {
+ private void writeRegioninfoOnFilesystem() throws IOException {
// Name of this file has two leading and trailing underscores so it doesn't
// clash w/ a store/family name. There is possibility, but assumption is
// that its slim (don't want to use control character in filename because
http://git-wip-us.apache.org/repos/asf/hbase/blob/451e39bf/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
index 1bbdd34..ee9098f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
@@ -123,10 +123,6 @@ public class TestHBaseFsckFixRegionInfo {
// rewrite the original .regioninfo
hriModified.writeToDisk(conf);
- System.out.println("SUCCESSFULLY CORRUPTED:"
- + !hriOriginal.toString().equals(hriModified.toString()));
- System.out.println("Original " + hriOriginal.toString());
- System.out.println("Modified: " + hriModified.toString());
modifyCount++;
}
}
[47/50] [abbrv] git commit: [HBASE-11602] Kill itself when election
znode is deleted
Posted by ec...@apache.org.
[HBASE-11602] Kill itself when election znode is deleted
Summary: In case the election znode is deleted (manually for example), the active master should kill itself.
Test Plan: `TestMasterCheckZKNode`
Reviewers: manukranthk, aaiyer, gauravm, fan
Reviewed By: fan
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1459946
Tasks: 4717508
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43727 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7281aa16
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7281aa16
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7281aa16
Branch: refs/heads/0.89-fb
Commit: 7281aa1641857b08347abc20080e4557e8d1b3eb
Parents: 89949ea
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jul 28 20:06:36 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 6 +-
.../hbase/master/ZKMasterAddressWatcher.java | 18 +++-
.../hadoop/hbase/HBaseTestingUtility.java | 7 ++
.../hbase/master/TestMasterCheckZNode.java | 99 ++++++++++++++++++++
4 files changed, 123 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7281aa16/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 959a070..b407783 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -884,7 +884,8 @@ HMasterRegionInterface, Watcher, StoppableMaster {
/* Main processing loop */
FINISHED: while (!this.closed.get()) {
// check if we should be shutting down
- if (clusterShutdownRequested.get()) {
+ if (zkMasterAddressWatcher.isElectionZNodeDeleted()
+ || clusterShutdownRequested.get()) {
// The region servers won't all exit until we stop scanning the
// meta regions
this.regionManager.stopScanners();
@@ -1682,7 +1683,8 @@ HMasterRegionInterface, Watcher, StoppableMaster {
List<Future<Void>> futures = new ArrayList<>();
for (final Pair<HRegionInfo, HServerAddress> entry : tableRegions) {
Callable<Void> writer = new Callable<Void>() {
-
+
+ @Override
public Void call() throws IOException {
HRegionInfo hri = entry.getFirst();
hri.writeToDisk(conf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/7281aa16/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java b/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
index 75d48e5..5af0fda 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
@@ -19,17 +19,16 @@
*/
package org.apache.hadoop.hbase.master;
+import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
-import java.util.concurrent.atomic.AtomicBoolean;
-
/**
* ZooKeeper watcher for the master address. Also watches the cluster state
@@ -45,6 +44,7 @@ class ZKMasterAddressWatcher implements Watcher {
private ZooKeeperWrapper zookeeper;
private final StoppableMaster stoppable;
+ private final AtomicBoolean electionZNodeDeleted = new AtomicBoolean(false);
/**
* Create this watcher using passed ZooKeeperWrapper instance.
@@ -70,6 +70,7 @@ class ZKMasterAddressWatcher implements Watcher {
this.stoppable.requestClusterShutdown();
} else if (event.getPath().equals(this.zookeeper.masterElectionZNode)){
LOG.info("Master address ZNode deleted, notifying waiting masters");
+ electionZNodeDeleted.set(true);
notifyAll();
}
} else if(type.equals(EventType.NodeCreated)) {
@@ -119,7 +120,8 @@ class ZKMasterAddressWatcher implements Watcher {
LOG.debug("Won't start Master because of requested shutdown");
return false;
}
- if(this.zookeeper.writeMasterAddress(address)) {
+ if (this.zookeeper.writeMasterAddress(address)) {
+ electionZNodeDeleted.set(false);
this.zookeeper.setClusterState(true);
this.zookeeper.setClusterStateWatch();
// Watch our own node
@@ -142,4 +144,10 @@ class ZKMasterAddressWatcher implements Watcher {
notifyAll();
}
-}
\ No newline at end of file
+ /**
+ * Returns whether the election znode was deleted.
+ */
+ public boolean isElectionZNodeDeleted() {
+ return electionZNodeDeleted.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7281aa16/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index a237776..b161f51 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2056,6 +2056,13 @@ public class HBaseTestingUtility {
}
/**
+ * Sets the configuration to use local-file-system.
+ */
+ public void useLFS() {
+ this.conf.set(FS_TYPE_KEY, FS_TYPE_LFS);
+ }
+
+ /**
* Moves the region to a new server and change the assignment accordingly.
*
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/7281aa16/src/test/java/org/apache/hadoop/hbase/master/TestMasterCheckZNode.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterCheckZNode.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterCheckZNode.java
new file mode 100644
index 0000000..b602225
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterCheckZNode.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright 2014 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Testcases related to znode checking.
+ */
+public class TestMasterCheckZNode {
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+
+ @Before
+ public void setUp() throws Exception {
+ TEST_UTIL.useLFS();
+ TEST_UTIL.startMiniCluster(2, 1);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * This testcase tries to remove the election znode to make both master
+ * running. The original active master should kill itself.
+ */
+ @Test(timeout = 300000)
+ public void testCheckingOfZKNode() throws Exception {
+ ZooKeeperWrapper zk =
+ ZooKeeperWrapper.createInstance(TEST_UTIL.getConfiguration(),
+ "testCheckingOfZKNode");
+ HMaster primary = null;
+ HMaster secondary = null;
+ for (HMaster master : TEST_UTIL.getMiniHBaseCluster().getMasters()) {
+ if (master.isActiveMaster()) {
+ primary = master;
+ } else {
+ secondary = master;
+ }
+ }
+ Assert.assertNotNull("Cannot find primary master", primary);
+ Assert.assertNotNull("Cannot find secondary master", secondary);
+
+ // Delete the election znode to make HBase crazy
+ zk.deleteMasterAddress();
+
+ // Wait for the original primary to die
+ long start = System.currentTimeMillis();
+ while (true) {
+ if (primary.isClosed()) {
+ break;
+ }
+ if (System.currentTimeMillis() - start > 30 * 1000) {
+ throw new Exception("The original primary doesn't kill itself!");
+ }
+ Thread.sleep(100);
+ }
+
+ // Wait for the secondary to be also active
+ start = System.currentTimeMillis();
+ while (true) {
+ if (secondary.isActiveMaster()) {
+ break;
+ }
+ if (System.currentTimeMillis() - start > 120 * 1000) {
+ throw new Exception("The secondary is not started!");
+ }
+ Thread.sleep(100);
+ }
+
+ Assert.assertTrue("new active master should be still alive",
+ secondary.isAlive() && secondary.isActiveMaster());
+ }
+
+}
[41/50] [abbrv] git commit: [HBASE-11416] Fix profiling after adding
size aware server
Posted by ec...@apache.org.
[HBASE-11416] Fix profiling after adding size aware server
Summary:
After adding the size aware thrift server per-request
profiling was broken as it uses thread locals.
Test Plan: mvn clean test -Dtest=TestPerRequestProfiling
Reviewers: daviddeng, adela, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1421670
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43344 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/78c632a8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/78c632a8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/78c632a8
Branch: refs/heads/0.89-fb
Commit: 78c632a8d72ecebf0d8dfd9246c43a2e799467be
Parents: 451e39b
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jul 7 21:08:24 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../hbase/regionserver/ThriftHRegionServer.java | 123 ++++++++++++++++---
1 file changed, 109 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/78c632a8/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
index 0d6eaca..1ba2f62 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.TMultiResponse;
import org.apache.hadoop.hbase.client.TRowMutations;
import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram;
import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram.Bucket;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.ScannerResult;
import org.apache.hadoop.hbase.ipc.ThriftHRegionInterface;
import org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException;
@@ -107,8 +108,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<HRegionInfo> getRegionInfo(final byte[] regionName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<HRegionInfo>() {
@Override public HRegionInfo call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
HRegionInfo r = server.getRegionInfo(regionName);
LOG.debug("Printing the result of getClosestRowOrBefore : " + r);
@@ -123,9 +126,11 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Result> getClosestRowBefore(final byte[] regionName, final byte[] row,
final byte[] family) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Result>(){
@Override public Result call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
Result r = server.getClosestRowBefore(regionName, row, family);
if (r == null) {
@@ -143,8 +148,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> flushRegion(final byte[] regionName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.flushRegion(regionName);
} catch (Exception e) {
@@ -157,8 +164,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> flushRegion(final byte[] regionName, final long ifOlderThanTS) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.flushRegion(regionName, ifOlderThanTS);
} catch (IllegalArgumentException e) {
@@ -173,8 +182,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Long> getLastFlushTime(final byte[] regionName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Long>() {
@Override public Long call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.getLastFlushTime(regionName);
}
});
@@ -182,8 +193,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Map<byte[], Long>> getLastFlushTimes() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Map<byte[], Long>>() {
@Override public Map<byte[], Long> call() throws Exception {
+ HRegionServer.callContext.set(call);
MapWritable mapWritable = server.getLastFlushTimes();
Map<byte[], Long> map = new HashMap<>();
@@ -198,8 +211,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Long> getCurrentTimeMillis() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Long>() {
@Override public Long call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.getCurrentTimeMillis();
}
});
@@ -207,8 +222,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Long> getStartCode() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Long>() {
@Override public Long call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.getStartCode();
}
});
@@ -216,8 +233,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<String>> getStoreFileList(final byte[] regionName, final byte[] columnFamily) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<String>>() {
@Override public List<String> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getStoreFileList(regionName, columnFamily);
} catch (IllegalArgumentException e) {
@@ -230,8 +249,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<String>> getStoreFileListForColumnFamilies(final byte[] regionName,
final List<byte[]> columnFamilies) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<String>>() {
@Override public List<String> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
byte[][] columnFamiliesArray = new byte[columnFamilies.size()][];
return server.getStoreFileList(regionName, columnFamilies.toArray(columnFamiliesArray));
@@ -244,8 +265,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<String>> getStoreFileListForAllColumnFamilies(final byte[] regionName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<String>>() {
@Override public List<String> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getStoreFileList(regionName);
} catch (IllegalArgumentException e) {
@@ -257,8 +280,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<String>> getHLogsList(final boolean rollCurrentHLog) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<String>>() {
@Override public List<String> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getHLogsList(rollCurrentHLog);
} catch (IOException e) {
@@ -270,8 +295,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Result> get(final byte[] regionName, final Get get) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Result>() {
@Override public Result call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.get(regionName, get);
} catch (IOException e) {
@@ -284,8 +311,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<Result>> getRows(final byte[] regionName, final List<Get> gets) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<Result>>() {
@Override public List<Result> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
List<Result> resultList = new ArrayList<>();
Collections.addAll(resultList, server.get(regionName, gets));
@@ -299,8 +328,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Boolean> exists(final byte[] regionName, final Get get){
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.exists(regionName, get);
} catch (IOException e) {
@@ -313,9 +344,11 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> put(final byte[] regionName, final Put put) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.put(regionName, put);
} catch (IOException e) {
@@ -328,8 +361,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Integer> putRows(final byte[] regionName, final List<Put> puts) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Integer>() {
@Override public Integer call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.put(regionName, puts);
} catch (IOException e) {
@@ -341,8 +376,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> processDelete(final byte[] regionName, final Delete delete) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.delete(regionName, delete);
} catch (IOException e) {
@@ -355,8 +392,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Integer> processListOfDeletes(final byte[] regionName, final List<Delete> deletes) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Integer>() {
@Override public Integer call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.delete(regionName, deletes);
} catch (IOException e) {
@@ -369,9 +408,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Boolean> checkAndPut(final byte[] regionName, final byte[] row, final byte[] family,
final byte[] qualifier, final byte[] value, final Put put) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.checkAndPut(regionName, row, family, qualifier, value, put);
} catch (IOException e) {
@@ -384,9 +424,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Boolean> checkAndDelete(final byte[] regionName, final byte[] row,
final byte[] family, final byte[] qualifier, final byte[] value, final Delete delete) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.checkAndDelete(regionName, row, family, qualifier, value,
delete);
@@ -400,9 +441,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Long> incrementColumnValue(final byte[] regionName, final byte[] row,
final byte[] family, final byte[] qualifier, final long amount, final boolean writeToWAL) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Long>() {
@Override public Long call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.incrementColumnValue(regionName, row, family, qualifier, amount, writeToWAL);
} catch (IOException e) {
@@ -414,8 +456,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Long> openScanner(final byte[] regionName, final Scan scan) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Long>() {
@Override public Long call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.openScanner(regionName, scan);
} catch (IOException e) {
@@ -427,9 +471,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> mutateRow(final byte[] regionName, final TRowMutations arm) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.mutateRow(regionName, RowMutations.Builder.createFromTRowMutations(arm));
} catch (IOException e) {
@@ -442,8 +487,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> mutateRows(final byte[] regionName, final List<TRowMutations> armList) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
List<RowMutations> rowMutations = new ArrayList<>();
for (TRowMutations mutation : armList) {
@@ -461,8 +508,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
@Deprecated
public ListenableFuture<Result> next(final long scannerId) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Result>() {
@Override public Result call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.next(scannerId);
} catch (IOException e) {
@@ -474,13 +523,15 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<Result>> nextRows(final long scannerId, final int numberOfRows) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<Result>>() {
@Override public List<Result> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
Result[] result = server.nextInternal(scannerId, numberOfRows);
List<Result> resultList = new ArrayList<>(result.length);
- for (int i = 0; i < result.length; i ++) {
- resultList.add(addThriftRegionInfoQualifierIfNeeded(result[i]));
+ for (Result aResult : result) {
+ resultList.add(addThriftRegionInfoQualifierIfNeeded(aResult));
}
return resultList;
} catch (IOException e) {
@@ -492,8 +543,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> close(final long scannerId) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.close(scannerId);
} catch (IOException e) {
@@ -506,8 +559,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<RowLock> lockRow(final byte[] regionName, final byte[] row) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<RowLock>() {
@Override public RowLock call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return new RowLock(row, server.lockRow(regionName, row));
} catch (IOException e) {
@@ -519,8 +574,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> unlockRow(final byte[] regionName, final long lockId) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.unlockRow(regionName, lockId);
} catch (IOException e) {
@@ -533,9 +590,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<HRegionInfo>> getRegionsAssignment() {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<HRegionInfo>>() {
@Override public List<HRegionInfo> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return Arrays.asList(server.getRegionsAssignment());
} catch (IOException e) {
@@ -547,8 +605,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<HServerInfo> getHServerInfo() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<HServerInfo>() {
@Override public HServerInfo call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getHServerInfo();
} catch (IOException e) {
@@ -560,8 +620,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<TMultiResponse> multiAction(final MultiAction multi) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<TMultiResponse>() {
@Override public TMultiResponse call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return TMultiResponse.Builder.createFromMultiResponse(server
.multiAction(multi));
@@ -574,9 +636,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<MultiPutResponse> multiPut(final MultiPut puts) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<MultiPutResponse>() {
@Override public MultiPutResponse call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.multiPut(puts);
} catch (IOException e) {
@@ -589,8 +652,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> bulkLoadHFile(final String hfilePath, final byte[] regionName,
final byte[] familyName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.bulkLoadHFile(hfilePath, regionName, familyName);
} catch (IOException e) {
@@ -604,8 +669,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> bulkLoadHFile(final String hfilePath, final byte[] regionName,
final byte[] familyName, final boolean assignSeqNum) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return writeService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.bulkLoadHFile(hfilePath, regionName, familyName, assignSeqNum);
} catch (IOException e) {
@@ -618,9 +685,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> closeRegion(final HRegionInfo hri, final boolean reportWhenCompleted) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
server.closeRegion(hri, reportWhenCompleted);
} catch (IOException e) {
@@ -633,8 +701,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Integer> updateFavoredNodes(final AssignmentPlan plan) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Integer>() {
@Override public Integer call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.updateFavoredNodes(plan);
} catch (IOException e) {
@@ -646,8 +716,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> updateConfiguration() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
server.updateConfiguration();
return null;
}
@@ -656,8 +728,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> stop(final String why) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
server.stop(why);
return null;
}
@@ -666,8 +740,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<String> getStopReason() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<String>() {
@Override public String call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.getStopReason();
}
});
@@ -675,8 +751,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> setNumHDFSQuorumReadThreads(final int maxThreads) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
server.setNumHDFSQuorumReadThreads(maxThreads);
return null;
}
@@ -692,8 +770,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Boolean> isStopped() {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.isStopped();
}
});
@@ -701,9 +781,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Void> stopForRestart() {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Void>() {
@Override public Void call() throws Exception {
+ HRegionServer.callContext.set(call);
server.stopForRestart();
return null;
}
@@ -717,8 +798,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<String> getConfProperty(final String paramName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<String>() {
@Override public String call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.getConfProperty(paramName);
}
});
@@ -726,8 +809,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<Bucket>> getHistogram(final byte[] regionName) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<Bucket>>() {
@Override public List<Bucket> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
List<Bucket> buckets = server.getHistogram(regionName);
if (buckets == null) return new ArrayList<Bucket>();
@@ -741,8 +826,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<Bucket>> getHistogramForStore(final byte[] regionName, final byte[] family) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<Bucket>>() {
@Override public List<Bucket> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
List<Bucket> buckets = server.getHistogramForStore(regionName, family);
if (buckets == null) return new ArrayList<Bucket>();
@@ -758,9 +845,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
public ListenableFuture<byte[]> callEndpoint(final String epName, final String methodName,
final List<byte[]> params, final byte[] regionName, final byte[] startRow,
final byte[] stopRow) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<byte[]>() {
@Override public byte[] call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.callEndpoint(epName, methodName, params, regionName,
startRow, stopRow);
@@ -773,8 +861,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<List<List<Bucket>>> getHistograms(final List<byte[]> regionNames) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<List<List<Bucket>>>() {
@Override public List<List<Bucket>> call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getHistograms(regionNames);
} catch (IOException e) {
@@ -787,9 +877,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<HRegionLocation> getLocation(
final byte[] table, final byte[] row, final boolean reload) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<HRegionLocation>() {
@Override public HRegionLocation call() throws Exception {
+ HRegionServer.callContext.set(call);
try {
return server.getLocation(table, row, reload);
} catch (IOException e) {
@@ -802,9 +893,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<ScannerResult> scanOpen(
final byte[] regionName, final Scan scan, final int numberOfRows) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<ScannerResult>() {
@Override public ScannerResult call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.scanOpen(regionName, scan, numberOfRows);
}
});
@@ -812,9 +904,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<ScannerResult> scanNext(final long id, final int numberOfRows) {
-
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<ScannerResult>() {
@Override public ScannerResult call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.scanNext(id, numberOfRows);
}
});
@@ -822,8 +915,10 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
@Override
public ListenableFuture<Boolean> scanClose(final long id) {
+ final HBaseServer.Call call = HRegionServer.callContext.get();
return readService.submit(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
+ HRegionServer.callContext.set(call);
return server.scanClose(id);
}
});
[29/50] [abbrv] git commit: [HBASE-11233] Asynchronous multiputs
Posted by ec...@apache.org.
[HBASE-11233] Asynchronous multiputs
Summary:
Idea is to split the multiput pipeline into 2 parts :
1. Before the wall edit.
2. After the wall edit.
This diff attempts to perform actions after the wal edit in an asynchronous manner as a piggy back to the wal edit event. We can configure where the edit actually happens.
Test Plan: Representation diff. Still need to work on the tests. This diff tries to present the idea that I was pursueing. Extension to https://phabricator.fb.com/D1316543
Reviewers: aaiyer, liyintang, daviddeng, gauravm, adela, fan, elliott, arjen, rshroff
Reviewed By: elliott
Subscribers: mycnyc, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1333824
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43142 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c22acc7f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c22acc7f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c22acc7f
Branch: refs/heads/0.89-fb
Commit: c22acc7fa16028ff2fdcb7fcc9a2ac91221cb22f
Parents: 75ed75d
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 23:25:12 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hbase/master/TimeToLiveLogCleaner.java | 5 +-
.../hadoop/hbase/regionserver/HRegion.java | 266 +++++++++++++------
.../hbase/regionserver/HRegionServer.java | 102 ++++---
.../MultiVersionConsistencyControl.java | 3 -
.../hadoop/hbase/regionserver/wal/HLog.java | 223 ++++++++++------
.../hadoop/hbase/HBaseTestingUtility.java | 4 +
.../hadoop/hbase/client/TestAsyncMultiputs.java | 185 +++++++++++++
.../org/apache/hadoop/hbase/client/TestHCM.java | 9 +-
.../hadoop/hbase/master/TestOldLogsCleaner.java | 5 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 9 +-
.../wal/TestAsyncMultiputsFailingWal.java | 58 ++++
.../hadoop/hbase/regionserver/wal/TestHLog.java | 52 +++-
.../hbase/regionserver/wal/TestHLogSplit.java | 6 +-
.../wal/TestLogActionsListener.java | 2 +-
.../hbase/regionserver/wal/TestWALReplay.java | 9 +-
15 files changed, 708 insertions(+), 230 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
index cc9e571..fce153e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
@@ -48,8 +48,9 @@ public class TimeToLiveLogCleaner implements LogCleanerDelegate {
long currentTime = System.currentTimeMillis();
try {
// If the path name is in hourly format, skip getting modification time
- if (OldLogsCleaner.isOldLogsArchivedToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) {
- time = HLog.DATE_FORMAT.parse(filePath.getName()).getTime();
+ if (OldLogsCleaner.isOldLogsArchivedToHourlyDir()
+ && OldLogsCleaner.isMatchDatePattern(filePath)) {
+ time = HLog.getDateFormat().parse(filePath.getName()).getTime();
} else {
FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath);
time = fStat.getModificationTime();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a68f1c3..de8ba0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -56,6 +56,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nullable;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -106,9 +108,14 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
/**
* HRegion stores data for a certain region of a table. It stores all columns
@@ -1974,7 +1981,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
seqNum = this.log.append(regionInfo,
- regionInfo.getTableDesc().getName(), walEdit, now);
+ regionInfo.getTableDesc().getName(), walEdit, now).checkedGet();
}
// Now make changes to the memstore.
@@ -2055,7 +2062,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
try {
// All edits for the given row (across all column families) must happen atomically.
- put(put.getFamilyMap(), writeToWAL);
+ put(put.getFamilyMap(), writeToWAL).checkedGet();
} finally {
if (lockid == null) releaseRowLock(lid);
}
@@ -2090,14 +2097,28 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* Perform a batch put with no pre-specified locks
* @see HRegion#put(Pair[])
*/
- public OperationStatusCode[] put(Put[] puts) throws IOException {
+ public CheckedFuture<OperationStatusCode[], IOException> put(Put[] puts) throws IOException {
@SuppressWarnings("unchecked")
Pair<Mutation, Integer> putsAndLocks[] = new Pair[puts.length];
for (int i = 0; i < puts.length; i++) {
putsAndLocks[i] = new Pair<Mutation, Integer>(puts[i], null);
}
- return batchMutateWithLocks(putsAndLocks, "multiput_");
+ return Futures.makeChecked(batchMutateWithLocks(putsAndLocks, "multiput_"),
+ getExceptionMapperFunction());
+ }
+
+ public Function<Exception, IOException> getExceptionMapperFunction() {
+ return new Function<Exception, IOException>() {
+ @Override
+ @Nullable
+ public IOException apply(@Nullable Exception e) {
+ if (e instanceof ExecutionException) {
+ return new IOException(e.getCause());
+ }
+ return new IOException(e);
+ }
+ };
}
/**
@@ -2106,33 +2127,55 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* @param methodName "multiput_/multidelete_" to update metrics correctly.
* @throws IOException
*/
- public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
- String methodName) throws IOException {
+ public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
+ Pair<Mutation, Integer>[] putsAndLocks, String methodName)
+ throws IOException {
BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
new BatchOperationInProgress<Pair<Mutation,Integer>>(putsAndLocks);
+ return batchMutateWithLazyRetry(batchOp, methodName);
+ }
- while (!batchOp.isDone()) {
+ public ListenableFuture<OperationStatusCode[]> batchMutateWithLazyRetry(
+ final BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
+ final String methodName) throws IOException {
+
+ if (!batchOp.isDone()) {
checkReadOnly();
checkResources();
- long newSize;
splitsAndClosesLock.readLock().lock();
try {
doPreMutationHook(batchOp);
- long addedSize = doMiniBatchOp(batchOp, methodName);
- newSize = this.incMemoryUsage(addedSize);
+ return Futures.transform(doMiniBatchOp(batchOp, methodName),
+ new AsyncFunction<Long, OperationStatusCode[]>() {
+ @Override
+ public ListenableFuture<OperationStatusCode[]> apply(Long addedSize)
+ throws Exception {
+ splitsAndClosesLock.readLock().lock();
+ try {
+ long newSize = incMemoryUsage(addedSize);
+ if (isFlushSize(newSize)) {
+ requestFlush();
+ }
+ } finally {
+ splitsAndClosesLock.readLock().unlock();
+ }
+ if (batchOp.isDone()) {
+ HRegion.writeOps.incrementAndGet();
+ return Futures.immediateFuture(batchOp.retCodes);
+ }
+ return batchMutateWithLazyRetry(batchOp, methodName);
+ }
+ });
+ } catch (Exception e) {
+ return Futures.immediateFuture(batchOp.retCodes);
} finally {
splitsAndClosesLock.readLock().unlock();
}
- if (isFlushSize(newSize)) {
- requestFlush();
- }
}
- HRegion.writeOps.incrementAndGet();
- return batchOp.retCodes;
+ return Futures.immediateFuture(batchOp.retCodes);
}
-
/**
* Execute coprocessor hooks for mutations
* TODO: adela currently can work only with prePut and we need to implement the rest..
@@ -2165,23 +2208,26 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
- private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
- String methodNameForMetricsUpdate) throws IOException {
+ private ListenableFuture<Long> doMiniBatchOp(
+ final BatchOperationInProgress <Pair<Mutation, Integer>> batchOp,
+ final String methodNameForMetricsUpdate)
+ throws IOException {
+ boolean preWallEditSuccessfull = false;
String signature = null;
// variable to note if all Put items are for the same CF -- metrics related
boolean isSignatureClear = true;
- long now = EnvironmentEdgeManager.currentTimeMillis();
+ final long now = EnvironmentEdgeManager.currentTimeMillis();
byte[] byteNow = Bytes.toBytes(now);
boolean locked = false;
/** Keep track of the locks we hold so we can release them in finally clause */
- List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
+ final List<Integer> acquiredLocks =
+ Lists.newArrayListWithCapacity(batchOp.operations.length);
// We try to set up a batch in the range [firstIndex,lastIndexExclusive)
- int firstIndex = batchOp.nextIndexToProcess;
+ final int firstIndex = batchOp.nextIndexToProcess;
int lastIndexExclusive = firstIndex;
- boolean success = false;
try {
// ------------------------------------
// STEP 1. Try to acquire as many locks as we can, and ensure
@@ -2193,7 +2239,8 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
Integer currentLockID = null;
while (lastIndexExclusive < batchOp.operations.length) {
- Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
+ Pair<Mutation, Integer> nextPair =
+ batchOp.operations[lastIndexExclusive];
Mutation op = nextPair.getFirst();
Integer providedLockId = nextPair.getSecond();
@@ -2244,6 +2291,9 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
}
+
+ final int finalLastIndexExclusive = lastIndexExclusive;
+
// We've now grabbed as many puts off the list as we can
assert numReadyToWrite > 0;
@@ -2253,7 +2303,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// ------------------------------------
// STEP 2. Update any LATEST_TIMESTAMP timestamps
// ----------------------------------
- for (int i = firstIndex; i < lastIndexExclusive; i++) {
+ for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
Mutation op = batchOp.operations[i].getFirst();
if (op instanceof Put) {
@@ -2267,13 +2317,14 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
}
- long seqNum = -1;
+ ListenableFuture<Long> wallEditFuture =
+ Futures.immediateFuture(-1L);
// ------------------------------------
// STEP 3. Write to WAL
// ----------------------------------
if (!this.disableWAL) {
WALEdit walEdit = new WALEdit();
- for (int i = firstIndex; i < lastIndexExclusive; i++) {
+ for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
// Skip puts that were determined to be invalid during preprocessing
if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
@@ -2283,48 +2334,80 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
}
// Append the edit to WAL
- seqNum = this.log.append(regionInfo,
- regionInfo.getTableDesc().getName(),
- walEdit, now);
- }
-
- // ------------------------------------
- // STEP 4. Write back to memstore
- // ----------------------------------
-
- long addedSize = 0;
- for (int i = firstIndex; i < lastIndexExclusive; i++) {
- if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
-
- Mutation op = batchOp.operations[i].getFirst();
- addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), seqNum);
- batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
+ wallEditFuture = this.log.append(regionInfo,
+ regionInfo.getTableDesc().getName(),
+ walEdit, now);
}
- success = true;
- return addedSize;
+ final String finalSignature = signature;
+
+ preWallEditSuccessfull = true;
+
+ return Futures.transform(wallEditFuture, new AsyncFunction<Long, Long>() {
+ @Override
+ public ListenableFuture<Long> apply(Long input) throws Exception {
+ boolean memstoreUpdatesSuccessful = false;
+ updatesLock.readLock().lock();
+ try {
+ // ------------------------------------
+ // STEP 4. Write back to memstore
+ // ----------------------------------
+
+ long addedSize = 0;
+ for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
+ if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
+
+ Mutation op = batchOp.operations[i].getFirst();
+ addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), input);
+ batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
+ }
+ memstoreUpdatesSuccessful = true;
+ return Futures.immediateFuture(addedSize);
+ } finally {
+ releaseLocksAndUpdateMetrics(updatesLock, acquiredLocks,
+ finalSignature, methodNameForMetricsUpdate, firstIndex,
+ finalLastIndexExclusive, batchOp, true,
+ memstoreUpdatesSuccessful, now);
+ }
+ }
+ });
+ } catch (Throwable t) {
+ return Futures.immediateFuture(null);
} finally {
- if (locked) {
- this.updatesLock.readLock().unlock();
- }
-
- releaseRowLocks(acquiredLocks);
+ releaseLocksAndUpdateMetrics(updatesLock,
+ Collections.<Integer> emptyList(), signature,
+ methodNameForMetricsUpdate, firstIndex, lastIndexExclusive, batchOp,
+ locked, preWallEditSuccessfull, now);
+ }
+ }
- // do after lock
- long after = EnvironmentEdgeManager.currentTimeMillis();
- if (null == signature) {
- signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
- }
- HRegion.incrTimeVaryingMetric(signature + methodNameForMetricsUpdate, after - now);
+ /**
+ * This method was intended for code de-duplication and nothing else.
+ */
+ private void releaseLocksAndUpdateMetrics(ReentrantReadWriteLock updatedLock,
+ List<Integer> acquiredLocks, String signature,
+ String methodNameForMetricsUpdate, int firstIndex, int lastIndexExclusive,
+ BatchOperationInProgress<Pair<Mutation, Integer>> batchOp, boolean locked,
+ boolean preWallEditSuccessfull, long now) {
+ if (locked) {
+ this.updatesLock.readLock().unlock();
+ }
+ releaseRowLocks(acquiredLocks);
+ // do after lock
+ long after = EnvironmentEdgeManager.currentTimeMillis();
+ if (null == signature) {
+ signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
+ }
+ HRegion.incrTimeVaryingMetric(
+ signature + methodNameForMetricsUpdate, after - now);
- if (!success) {
- for (int i = firstIndex; i < lastIndexExclusive; i++) {
- if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
- batchOp.retCodes[i] = OperationStatusCode.FAILURE;
- }
+ if (!preWallEditSuccessfull) {
+ for (int i = firstIndex; i < lastIndexExclusive; i++) {
+ if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
+ batchOp.retCodes[i] = OperationStatusCode.FAILURE;
}
}
- batchOp.nextIndexToProcess = lastIndexExclusive;
}
+ batchOp.nextIndexToProcess = lastIndexExclusive;
}
//TODO, Think that gets/puts and deletes should be refactored a bit so that
@@ -2385,7 +2468,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
if (matches) {
// All edits for the given row (across all column families) must happen atomically.
if (isPut) {
- put(((Put)w).getFamilyMap(), writeToWAL);
+ put(((Put)w).getFamilyMap(), writeToWAL).checkedGet();
} else {
Delete d = (Delete)w;
prepareDeleteFamilyMap(d);
@@ -2481,7 +2564,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
throws IOException {
Map<byte[], List<KeyValue>> familyMap = new HashMap<byte[], List<KeyValue>>();
familyMap.put(family, edits);
- this.put(familyMap, true);
+ this.put(familyMap, true).checkedGet();
}
/**
@@ -2491,11 +2574,12 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
* @param writeToWAL if true, then we should write to the log
* @throws IOException
*/
- private void put(final Map<byte [], List<KeyValue>> familyMap,
+ private CheckedFuture<Void, IOException> put(final Map<byte [], List<KeyValue>> familyMap,
boolean writeToWAL) throws IOException {
- long now = EnvironmentEdgeManager.currentTimeMillis();
+ final long now = EnvironmentEdgeManager.currentTimeMillis();
byte[] byteNow = Bytes.toBytes(now);
- boolean flush = false;
+ ListenableFuture<Long> wallEditSyncFuture =
+ Futures.immediateFuture(new Long(-1));
this.updatesLock.readLock().lock();
try {
checkFamilies(familyMap.keySet());
@@ -2506,30 +2590,43 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// If order is reversed, i.e. we write to memstore first, and
// for some reason fail to write/sync to commit log, the memstore
// will contain uncommitted transactions.
- long seqNum = -1;
if (!this.disableWAL && writeToWAL) {
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
- seqNum = this.log.append(regionInfo,
+ wallEditSyncFuture = this.log.append(regionInfo,
regionInfo.getTableDesc().getName(), walEdit, now);
}
+ return Futures.makeChecked(Futures.transform(wallEditSyncFuture,
+ new AsyncFunction<Long, Void>() {
+ @Override
+ public ListenableFuture<Void> apply(Long input) throws Exception {
+ boolean flush = false;
+ updatesLock.readLock().lock();
+ try {
+ long addedSize = applyFamilyMapToMemstore(familyMap, input);
+ flush = isFlushSize(incMemoryUsage(addedSize));
+ } finally {
+ updatesLock.readLock().unlock();
+ }
- long addedSize = applyFamilyMapToMemstore(familyMap, seqNum);
- flush = isFlushSize(this.incMemoryUsage(addedSize));
+ // do after lock
+ long after = EnvironmentEdgeManager.currentTimeMillis();
+ String signature = SchemaMetrics.generateSchemaMetricsPrefix(
+ getTableDesc().getNameAsString(), familyMap.keySet());
+ HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
+
+ if (flush) {
+ // Request a cache flush. Do it outside update lock.
+ requestFlush();
+ }
+ return Futures.immediateFuture(null);
+ }
+ }), this.getExceptionMapperFunction());
+ } catch (Exception e) {
+ return Futures.immediateFailedCheckedFuture(new IOException(e));
} finally {
this.updatesLock.readLock().unlock();
}
-
- // do after lock
- long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = SchemaMetrics.generateSchemaMetricsPrefix(
- this.getTableDesc().getNameAsString(), familyMap.keySet());
- HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
-
- if (flush) {
- // Request a cache flush. Do it outside update lock.
- requestFlush();
- }
}
/**
@@ -3788,8 +3885,9 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// 6. append/sync all edits at once
// TODO: Do batching as in doMiniBatchPut
- long seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
- walEdit, now);
+ long seqNum;
+ seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
+ walEdit, now).checkedGet();
// 7. apply to memstore
long addedSize = 0;
@@ -3875,7 +3973,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
WALEdit walEdit = new WALEdit();
walEdit.add(newKv);
seqNum = this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now);
+ walEdit, now).checkedGet();
}
// Now request the ICV to the store, this will set the timestamp
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7a5911b..3f3947d 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -64,6 +64,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nullable;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
@@ -174,7 +176,12 @@ import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
@@ -2064,6 +2071,13 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
return this.hlogs[i];
}
+ /**
+ * Return number of logs present in the regionserver
+ */
+ public int getLogCount() {
+ return this.hlogs.length;
+ }
+
public int getTotalHLogCnt() {
return this.hlogs.length;
}
@@ -2917,13 +2931,18 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
}
}
- @Override
- public int put(final byte[] regionName, final List<Put> puts)
- throws IOException {
+ public CheckedFuture<Integer, IOException> putAsync(final byte[] regionName,
+ final List<Put> puts) throws IOException {
return applyMutations(regionName, puts, "multiput_");
}
- private int applyMutations(final byte[] regionName,
+ @Override
+ public int put(final byte[] regionName,
+ final List<Put> puts) throws IOException {
+ return applyMutations(regionName, puts, "multiput_").checkedGet();
+ }
+
+ private CheckedFuture<Integer, IOException> applyMutations(final byte[] regionName,
final List<? extends Mutation> mutations,
String methodName)
throws IOException {
@@ -2944,18 +2963,45 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
opWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
}
- OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks,
+ ListenableFuture<OperationStatusCode[]> batchMutateFuture =
+ region.batchMutateWithLocks(opWithLocks,
methodName);
- for (i = 0; i < codes.length; i++) {
- if (codes[i] != OperationStatusCode.SUCCESS)
- return i;
- }
- return HConstants.MULTIPUT_SUCCESS;
+ return Futures.makeChecked(Futures.transform(batchMutateFuture,
+ new AsyncFunction<OperationStatusCode[], Integer>() {
+
+ @Override
+ public ListenableFuture<Integer> apply(OperationStatusCode[] codes)
+ throws Exception {
+ try {
+ for (int i = 0; i < codes.length; i++) {
+ if (codes[i] != OperationStatusCode.SUCCESS)
+ return Futures.immediateFuture(i);
+ }
+ return Futures.immediateFuture(HConstants.MULTIPUT_SUCCESS);
+ } catch (Throwable t) {
+ return Futures.immediateFailedCheckedFuture(
+ convertThrowableToIOE(cleanup(t)));
+ }
+ }
+ }), getConvertThrowableToIOEFunction());
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
}
}
+ private Function<Exception, IOException> getConvertThrowableToIOEFunction() {
+ return new Function<Exception, IOException>() {
+ @Override
+ @Nullable
+ public IOException apply(@Nullable Exception e) {
+ if (e instanceof ExecutionException) {
+ return convertThrowableToIOE(cleanup(e.getCause()));
+ }
+ return convertThrowableToIOE(cleanup(e));
+ }
+ };
+ }
+
private boolean checkAndMutate(final byte[] regionName, final byte [] row,
final byte [] family, final byte [] qualifier, final byte [] value,
final Writable w, Integer lock) throws IOException {
@@ -3218,7 +3264,7 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
public int delete(final byte[] regionName,
final List<Delete> deletes)
throws IOException {
- return applyMutations(regionName, deletes, "multidelete_");
+ return applyMutations(regionName, deletes, "multidelete_").checkedGet();
}
@Override
@@ -3867,32 +3913,18 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
public MultiPutResponse multiPut(MultiPut puts) throws IOException {
MultiPutResponse resp = new MultiPutResponse();
- // do each region as it's own.
- int size = puts.puts.size();
- int index = 0;
+ Map<byte[], CheckedFuture<Integer, IOException>> futuresMap =
+ new TreeMap<>(Bytes.BYTES_COMPARATOR);
for( Map.Entry<byte[], List<Put>> e: puts.puts.entrySet()) {
- int result = put(e.getKey(), e.getValue());
- resp.addResult(e.getKey(), result);
-
- index++;
- if (index < size) {
- // remove the reference to the region list of Puts to save RAM except
- // for the last one. We will lose the reference to the last one pretty
- // soon anyway; keep it for a little more, until we get back
- // to HBaseServer level, where we might need to pretty print
- // the MultiPut request for debugging slow/large puts.
- // Note: A single row "put" from client also end up in server
- // as a multiPut().
- // We set the value to an empty array so taskmonitor can either get
- // the old value or an empty array. If we call clear() on the array,
- // then we might have a race condition where we're iterating over the
- // array at the same time we clear it (which throws an exception)
- // This relies on the fact that Map.Entry.setValue() boils down to a
- // simple reference assignment, which is atomic
- e.setValue(emptyPutArray); // clear some RAM
- }
+ futuresMap.put(e.getKey(), putAsync(e.getKey(), e.getValue()));
+ }
+ for (Entry<byte[], CheckedFuture<Integer, IOException>> e :
+ futuresMap.entrySet()) {
+ CheckedFuture<Integer, IOException> f = e.getValue();
+ byte[] row = e.getKey();
+ int ret = f.checkedGet();
+ resp.addResult(row, ret);
}
-
return resp;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
index 6b28f03..53e9757 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
@@ -24,9 +24,6 @@ import java.util.LinkedList;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.Log;
-
/**
* Manages the read/write consistency within memstore. This provides
* an interface for readers to determine what entries to ignore, and
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 044fef4..7e2036e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -63,6 +63,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -80,13 +82,10 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.ipc.HBaseServer.Call;
-import org.apache.hadoop.hbase.ipc.ProfilingData;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionSeqidTransition;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -99,7 +98,13 @@ import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -146,7 +151,9 @@ public class HLog implements Syncable {
public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
public static final byte [] METAROW = Bytes.toBytes("METAROW");
public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
- public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH");
+ public static SimpleDateFormat getDateFormat() {
+ return new SimpleDateFormat("yyyy-MM-dd-HH");
+ }
/*
* Name of directory that holds recovered edits written by the wal log
@@ -181,8 +188,8 @@ public class HLog implements Syncable {
Collections.synchronizedList(new ArrayList<LogActionsListener>());
- static Class<? extends Writer> logWriterClass;
- static Class<? extends Reader> logReaderClass;
+ volatile static Class<? extends Writer> logWriterClass;
+ volatile static Class<? extends Reader> logReaderClass;
private OutputStream hdfs_out; // OutputStream associated with the current SequenceFile.writer
private int initialReplication; // initial replication factor of SequenceFile.writer
@@ -284,9 +291,6 @@ public class HLog implements Syncable {
private final List<LogEntryVisitor> logEntryVisitors =
new CopyOnWriteArrayList<LogEntryVisitor>();
- private volatile long lastLogRollStartTimeMillis = 0;
- private volatile long lastLogRollDurationMillis = 0;
-
/**
* Pattern used to validate a HLog file name
*/
@@ -305,9 +309,9 @@ public class HLog implements Syncable {
synchronized (this) {
min = Math.min(min, val);
max = Math.max(max, val);
+ total += val;
+ ++count;
}
- total += val;
- ++count;
}
Metric get() {
@@ -359,14 +363,17 @@ public class HLog implements Syncable {
*/
private class DoubleListBuffer {
private LinkedList<Entry> currentList = new LinkedList<Entry>();
+ private SettableFuture<Void> currentFuture = SettableFuture.create();;
private LinkedList<Entry> syncList = new LinkedList<Entry>();
/**
* Append a log entry into the buffer
* @param entry log entry
*/
- synchronized private void appendToBuffer(Entry entry) {
+ synchronized private ListenableFuture<Void> appendToBuffer(Entry entry) {
+ Preconditions.checkNotNull(this.currentFuture);
currentList.add(entry);
+ return this.currentFuture;
}
/**
@@ -375,6 +382,8 @@ public class HLog implements Syncable {
* @return number of log entries synced
*/
private int appendAndSync() throws IOException {
+ // The future returned to the append operation.
+ SettableFuture<Void> future;
synchronized (this) {
if (currentList.isEmpty()) { // no thing to sync
return 0;
@@ -385,26 +394,49 @@ public class HLog implements Syncable {
LinkedList<Entry> tmp = syncList;
syncList = currentList;
currentList = tmp;
+ future = currentFuture;
+ this.currentFuture = SettableFuture.create();
}
- // append entries to writer
- int syncedEntries = syncList.size();
- while (!syncList.isEmpty()) {
- Entry entry = syncList.remove();
- writer.append(entry);
- }
+ try {
+ // append entries to writer
+ int syncedEntries = syncList.size();
+ while (!syncList.isEmpty()) {
+ Entry entry = syncList.remove();
+ writer.append(entry);
+ }
- // sync the data
- long now = System.currentTimeMillis();
- writer.sync();
- syncTime.inc(System.currentTimeMillis() - now);
- return syncedEntries;
+ // sync the data
+ long now = System.currentTimeMillis();
+ writer.sync();
+ syncTime.inc(System.currentTimeMillis() - now);
+ future.set(null);
+ return syncedEntries;
+ } catch (Exception e) {
+ future.setException(e);
+ return 0;
+ }
}
}
private DoubleListBuffer logBuffer = new DoubleListBuffer();
/**
+ * Minimal constructor to get going in tests
+ */
+ public HLog(final Configuration conf, Path basePath) throws IOException {
+ this(FileSystem.get(conf), new Path(basePath, ".logs"),
+ new Path(basePath, ".oldlogs"), conf, null);
+ }
+
+ /**
+ * Minimal constructor to get going in tests
+ */
+ public HLog(final Configuration conf) throws IOException {
+ this(conf, new Path("/tmp"));
+ }
+
+ /**
* HLog creating with a null actions listener.
*
* @param fs filesystem handle
@@ -651,8 +683,6 @@ public class HLog implements Syncable {
this.numEntries.set(0);
t1 = EnvironmentEdgeManager.currentTimeMillis();
- lastLogRollStartTimeMillis = t0;
- lastLogRollDurationMillis = (t1 - t0);
}
Path oldFile = null;
@@ -699,7 +729,7 @@ public class HLog implements Syncable {
throws IOException {
try {
if (logReaderClass == null) {
- logReaderClass =conf.getClass("hbase.regionserver.hlog.reader.impl",
+ logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
SequenceFileLogReader.class, Reader.class);
}
@@ -708,8 +738,7 @@ public class HLog implements Syncable {
return reader;
} catch (IOException e) {
throw e;
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new IOException("Cannot get log reader", e);
}
}
@@ -1052,10 +1081,10 @@ public class HLog implements Syncable {
* of the memstore.
* @throws IOException
*/
- public long append(HRegionInfo info, byte [] tableName, WALEdit edits,
+ public CheckedFuture<Long, IOException> append(HRegionInfo info, byte[] tableName, WALEdit edits,
final long now) throws IOException {
if (!this.enabled || edits.isEmpty()) {
- return -1L;
+ return Futures.immediateCheckedFuture(-1L);
}
if (logSyncerThread.syncerShuttingDown) {
// can't acquire lock for the duration of append()
@@ -1063,13 +1092,13 @@ public class HLog implements Syncable {
throw new IOException("Cannot append; logSyncer shutting down");
}
- long len = edits.getTotalKeyValueLength();
- long txid = 0;
- long seqNum;
+ final long len = edits.getTotalKeyValueLength();
+ final long txid;
+ final long seqNum;
- long start = System.currentTimeMillis();
byte[] regionName = info.getRegionName();
+ ListenableFuture<Void> logSyncFuture;
synchronized (this.appendLock) {
// The 'lastSeqWritten' map holds the sequence number of the oldest
// write for each region (i.e. the first edit added to the particular
@@ -1080,51 +1109,54 @@ public class HLog implements Syncable {
this.firstSeqWrittenInCurrentMemstore.putIfAbsent(regionName, seqNum);
HLogKey logKey = makeKey(regionName, tableName, seqNum, now);
- doWrite(info, logKey, edits);
+ logSyncFuture = doWrite(info, logKey, edits);
// Only count 1 row as an unflushed entry.
txid = this.unflushedEntries.incrementAndGet();
}
+ if (info.isMetaRegion()) forceSync();
+ final long start = System.currentTimeMillis();
- // Update the metrics
- this.numEntries.incrementAndGet();
- writeSize.inc(len);
-
- // sync txn to file system
- start = System.nanoTime();
- this.sync(info.isMetaRegion(), txid);
-
- // Update the metrics and log down the outliers
- long end = System.nanoTime();
- long syncTime = end - start;
- gsyncTime.inc(syncTime);
- if (syncTime > SECOND_IN_NS) {
- LOG.warn(String.format(
- "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
- Thread.currentThread().getName(), syncTime, this.numEntries.get(),
- StringUtils.humanReadableInt(len)));
- slowSyncs.incrementAndGet();
- }
-
- if (slowSyncs.get() >= slowBeforeRoll) {
- this.requestLogRoll();
- this.slowSyncs.set(0);
- }
+ return Futures.makeChecked(Futures.transform(logSyncFuture,
+ new AsyncFunction<Void, Long>() {
+ @Override
+ public ListenableFuture<Long> apply(Void input) throws Exception {
+ Preconditions.checkArgument(txid >= getCurSyncPoint());
+ // Update the metrics
+ numEntries.incrementAndGet();
+ writeSize.inc(len);
+
+ // Update the metrics and log down the outliers
+ long end = System.currentTimeMillis();
+ long syncTime = end - start;
+ gsyncTime.inc(syncTime);
+
+ if (syncTime > SECOND_IN_NS) {
+ LOG.warn(String.format(
+ "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
+ Thread.currentThread().getName(), syncTime, numEntries.get(),
+ StringUtils.humanReadableInt(len)));
+ slowSyncs.incrementAndGet();
+ }
+ if (slowSyncs.get() >= slowBeforeRoll) {
+ requestLogRoll();
+ slowSyncs.set(0);
+ }
+ return Futures.immediateFuture(new Long(seqNum));
+ }
+ }), this.getExceptionMapperFunction());
+ }
- // Update the per-request profiling data
- Call call = HRegionServer.callContext.get();
- ProfilingData pData = call == null ? null : call.getProfilingData();
- if (pData != null) {
- if (this.lastLogRollStartTimeMillis > start
- && end > this.lastLogRollStartTimeMillis) {
- // We also had a log roll in between
- pData.addLong(ProfilingData.HLOG_ROLL_TIME_MS, this.lastLogRollDurationMillis);
- // Do not account for this as the sync time.
- syncTime = syncTime - this.lastLogRollDurationMillis;
+ public Function<Exception, IOException> getExceptionMapperFunction() {
+ return new Function<Exception, IOException>() {
+ @Override
+ @Nullable
+ public IOException apply(@Nullable Exception e) {
+ if (e instanceof ExecutionException) {
+ return new IOException(e.getCause());
+ }
+ return new IOException(e);
}
- // update sync time
- pData.addLong(ProfilingData.HLOG_SYNC_TIME_MS, syncTime);
- }
- return seqNum;
+ };
}
/**
@@ -1167,7 +1199,7 @@ public class HLog implements Syncable {
forceSync = false;
// wake up every 100ms to check if logsyncer has to shut down
queueEmpty.awaitNanos(100*1000000);
- if (unflushedEntries.get() == syncTillHere) {
+ if (unflushedEntries.get() >= syncTillHere) {
// call hflush() if we haven't flushed for a while now
// This force-sync is just a safety mechanism - we being
// paranoid. If there hasn't been any syncing activity for
@@ -1210,6 +1242,18 @@ public class HLog implements Syncable {
}
/**
+ * This method needs to be called to indicate that we need to force the
+ * log syncer thread to sync.
+ * @param force
+ */
+ public void setForceSync(boolean force) {
+ forceSync = false;
+ }
+
+ public long getCurSyncPoint() {
+ return syncTillHere;
+ }
+ /**
* This method first signals the thread that there's a sync needed
* and then waits for it to happen before returning.
* @throws IOException
@@ -1276,6 +1320,14 @@ public class HLog implements Syncable {
logSyncerThread.addToSyncQueue(force, txid);
}
+ public void forceSync() {
+ logSyncerThread.setForceSync(true);
+ }
+
+ public long getCurSyncPoint() {
+ return logSyncerThread.getCurSyncPoint();
+ }
+
private void hflush() {
synchronized (this.updateLock) {
if (this.closed) {
@@ -1379,9 +1431,9 @@ public class HLog implements Syncable {
}
}
- protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
+ protected ListenableFuture<Void> doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
throws IOException {
- this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
+ return this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
}
/** @return How many items have been added to the log */
@@ -1890,7 +1942,8 @@ public class HLog implements Syncable {
String subDirectoryName;
if (ARCHIVE_TO_HOURLY_DIR) {
// Group into hourly sub-directory
- subDirectoryName = DATE_FORMAT.format(Calendar.getInstance().getTime());
+ subDirectoryName =
+ getDateFormat().format(Calendar.getInstance().getTime());
} else {
// since the filename is a valid name, we know there
// is a last '.' (won't return -1)
@@ -2388,11 +2441,25 @@ public class HLog implements Syncable {
walEdit.add(kvLast);
walEdit.add(kvNext);
walEdit.add(kvRegionInfo);
- long newSeqid = append(regionInfo, regionInfo.getTableDesc().getName(), walEdit, now);
+ long newSeqid;
+ try {
+ newSeqid = append(regionInfo, regionInfo.getTableDesc()
+ .getName(), walEdit, now).get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new IOException(e);
+ }
LOG.info("Region " + regionInfo.getRegionNameAsString() +
" sequence id transition " +
seqidTransition.getLastSeqid() + "->" +
seqidTransition.getNextSeqid() + " appended to HLog at seqid=" + newSeqid);
return true;
}
+
+ /**
+ * Returns the underlying configuration object.
+ * @return
+ */
+ public Configuration getConfiguration() {
+ return this.conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 649512f..a237776 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2086,4 +2086,8 @@ public class HBaseTestingUtility {
getHBaseAdmin().moveRegion(regionInfo.getRegionName(),
regionServer.toString());
}
+
+ public HConnection getConnection() {
+ return HConnectionManager.getConnection(this.conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java b/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
new file mode 100644
index 0000000..3b0ef2b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
@@ -0,0 +1,185 @@
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestAsyncMultiputs {
+ private static final Log LOG = LogFactory.getLog(TestAsyncMultiputs.class);
+ private static final HBaseTestingUtility util = new HBaseTestingUtility();
+ private static int NUM_REGIONSERVERS = 1;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ util.startMiniCluster(NUM_REGIONSERVERS);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ util.shutdownMiniCluster();
+ }
+
+ /**
+ * Test intended to check if the wal gives the control
+ * back when writers are calling append.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testMultithreadedCallbackFromHLog() throws IOException, InterruptedException {
+ final HLog log = new HLog(util.getConfiguration(),
+ new Path("/tmp/testMultithreadedCallback"));
+ int numThreads = 100;
+ ExecutorService executor = Executors.newCachedThreadPool();
+ for (int i = 0; i < numThreads; i++) {
+ final int threadId = i;
+ executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HTableDescriptor desc =
+ new HTableDescriptor(Bytes.toBytes("table" + threadId));
+ HRegionInfo info = new HRegionInfo(desc, new byte[0], new byte[0]);
+ WALEdit edits = new WALEdit();
+ for (byte[] row : getRandomRows(1, 10)) {
+ edits.add(new KeyValue(row));
+ }
+ try {
+ log.append(info, desc.getName(), edits,
+ System.currentTimeMillis()).get(1, TimeUnit.MINUTES);
+ } catch(Exception e) {
+ fail();
+ }
+ return null;
+ }
+ });
+ }
+ executor.shutdown();
+ executor.awaitTermination(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testMultiputSucceeds() throws IOException {
+ String tableName = "testMultiputSucceeds";
+ byte[] tableNameBytes = Bytes.toBytes(tableName);
+ String cf = "cf";
+ byte[][] families = new byte[][] { Bytes.toBytes(cf) };
+ LOG.debug("Creating the table " + tableName);
+ int numRegions = 100;
+ HRegionServer server = createTableAndGiveMeTheRegionServer(tableNameBytes,
+ families, numRegions);
+ MultiPut multiput = getRandomMultiPut(tableNameBytes, 1000, families[0],
+ server.getServerInfo().getServerAddress());
+ server.multiPut(multiput);
+ for (HRegion region : server.getOnlineRegionsAsArray()) {
+ server.flushRegion(region.getRegionName());
+ }
+ for (Entry<byte[], List<Put>> entry : multiput.getPuts().entrySet()) {
+ byte[] regionName = entry.getKey();
+ for (Put p : entry.getValue()) {
+ Get g = new Get(p.getRow());
+ Result r = server.get(regionName, g);
+ assertNotNull(g);
+ assertTrue(Bytes.equals(r.getValue(families[0], null), p.getRow()));
+ assertTrue(Bytes.equals(r.getRow(), p.getRow()));
+ }
+ }
+ }
+
+ public HRegionServer createTableAndGiveMeTheRegionServer(
+ byte[] tableName, byte[][] families, int numregions) throws IOException {
+ LOG.debug("Creating table " + Bytes.toString(tableName) + " with "
+ + numregions + " regions");
+ util.createTable(tableName, families, 1, Bytes.toBytes("aaa"),
+ Bytes.toBytes("zzz"), numregions);
+ assertTrue(NUM_REGIONSERVERS == 1);
+ HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
+ LOG.debug("Found the regionserver. Returning" + rs);
+ return rs;
+ }
+
+ public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
+ HServerAddress addr) throws IOException {
+ return getRandomMultiPut(tableName, numPuts, cf, addr, util);
+ }
+
+ /**
+ * Creates random puts with rows between aaa.. to zzz..
+ * @param tableName
+ * @param numPuts
+ * @param cf
+ * @param addr
+ * @param util : The hbase testing utility to use.
+ * @return
+ * @throws IOException
+ */
+ public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
+ HServerAddress addr, HBaseTestingUtility util) throws IOException {
+ MultiPut multiput = new MultiPut(addr);
+ List<Put> puts = getRandomPuts(numPuts, cf);
+ LOG.debug("Creating " + numPuts + " puts to insert into the multiput");
+ HConnection conn = util.getConnection();
+ for (Put put : puts) {
+ byte[] regionName = conn.getRegionLocation(new StringBytes(tableName),
+ put.getRow(), false).getRegionInfo().getRegionName();
+ multiput.add(regionName, put);
+ }
+ return multiput;
+ }
+
+ /**
+ * Creates puts with rows between aaa... and zzz..
+ * @param numPuts
+ * @param cf
+ * @return
+ */
+ public static List<Put> getRandomPuts(int numPuts, byte[] cf) {
+ List<Put> ret = new ArrayList<Put> (numPuts);
+ List<byte[]> rows = getRandomRows(numPuts, 3);
+ for (byte[] row : rows) {
+ Put p = new Put(row);
+ p.add(cf, null, row);
+ ret.add(p);
+ }
+ return ret;
+ }
+
+ public static List<byte[]> getRandomRows(int numRows, int rowlen) {
+ Random rand = new Random();
+ List<byte[]> ret = new ArrayList<byte[]>(numRows);
+ for (int i = 0; i < numRows; i++) {
+ byte[] row = new byte[rowlen];
+ for (int j = 0; j < rowlen; j++) {
+ row[j] = (byte) ('a' + (byte) rand.nextInt('z' - 'a'));
+ }
+ ret.add(row);
+ }
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index f22f256..3f0c5bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -50,6 +50,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
/**
* This class is for testing HCM features
*/
@@ -275,9 +278,11 @@ public class TestHCM {
}
@Override
- public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
+ public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
+ Pair<Mutation, Integer>[] putsAndLocks,
String methodName) throws IOException {
- throw new IOException("Test induced failure");
+ return Futures.immediateFailedFuture(
+ new IOException("Test induced failure"));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
index 42b7880..758ae4e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
@@ -144,10 +144,11 @@ public class TestOldLogsCleaner {
fs.mkdirs(legacyDir);
fs.createNewFile(new Path(legacyDir, "123.456"));
Calendar cal = Calendar.getInstance();
- System.out.println("Now is: " + HLog.DATE_FORMAT.format(cal.getTime()));
+ System.out.println("Now is: " + HLog.getDateFormat().format(cal.getTime()));
for (int i = 0; i < 10; i++) {
cal.add(Calendar.HOUR, -1);
- Path hourDir = new Path(oldLogDir, HLog.DATE_FORMAT.format(cal.getTime()));
+ Path hourDir = new Path(oldLogDir,
+ HLog.getDateFormat().format(cal.getTime()));
fs.mkdirs(hourDir);
fs.createNewFile(new Path(hourDir, new Path(fakeMachineName + "." + i)));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 437c19c..cb4a9ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -372,7 +372,7 @@ public class TestHRegion extends HBaseTestCase {
puts[i].add(cf, qual, val);
}
- OperationStatusCode[] codes = this.region.put(puts);
+ OperationStatusCode[] codes = this.region.put(puts).get();
assertEquals(10, codes.length);
for (int i = 0; i < 10; i++) {
assertEquals(OperationStatusCode.SUCCESS, codes[i]);
@@ -382,7 +382,7 @@ public class TestHRegion extends HBaseTestCase {
LOG.info("Next a batch put with one invalid family");
puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
try {
- codes = this.region.put(puts);
+ codes = this.region.put(puts).get();
// put should throw an exception
assertTrue(false);
@@ -403,7 +403,7 @@ public class TestHRegion extends HBaseTestCase {
TestThread putter = new TestThread(ctx) {
@Override
public void doWork() throws IOException {
- retFromThread.set(region.put(puts));
+ retFromThread.set(region.put(puts).checkedGet());
}
};
LOG.info("...starting put thread while holding lock");
@@ -439,7 +439,8 @@ public class TestHRegion extends HBaseTestCase {
putsAndLocks.add(pair);
}
- codes = region.batchMutateWithLocks(putsAndLocks.toArray(new Pair[0]), "multiput_");
+ codes = region.batchMutateWithLocks(putsAndLocks.toArray(
+ new Pair[0]), "multiput_").get();
LOG.info("...performed put");
for (int i = 0; i < 10; i++) {
assertEquals(OperationStatusCode.SUCCESS, codes[i]);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
new file mode 100644
index 0000000..20150c0
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
@@ -0,0 +1,58 @@
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.MultiPut;
+import org.apache.hadoop.hbase.client.TestAsyncMultiputs;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.TestHLog.FailingSequenceFileLogWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+public class TestAsyncMultiputsFailingWal {
+ private final Log LOG = LogFactory.getLog(TestAsyncMultiputsFailingWal.class);
+
+ @Test
+ public void testFailedMultiput() throws Exception {
+ String tableName = "testFailedMultiput";
+ byte[] tableNameBytes = Bytes.toBytes(tableName);
+ String cf = "cf";
+ byte[][] families = new byte[][] { Bytes.toBytes(cf) };
+ LOG.debug("Creating the table " + tableName);
+ int numRegions = 3;
+ HBaseTestingUtility localutil = new HBaseTestingUtility();
+ localutil.startMiniCluster();
+
+ LOG.debug("Creating table " + tableName + " with "
+ + numRegions + " regions");
+ localutil.createTable(tableNameBytes, families, 1, Bytes.toBytes("aaa"),
+ Bytes.toBytes("zzz"), numRegions);
+ HRegionServer rs = localutil.getRSForFirstRegionInTable(tableNameBytes);
+ LOG.debug("Found the regionserver. Returning" + rs);
+
+ HLog.logWriterClass = null;
+ assertTrue(rs.getLogCount() == 1);
+ HLog log = rs.getLog(0);
+ log.getConfiguration().set("hbase.regionserver.hlog.writer.impl",
+ FailingSequenceFileLogWriter.class.getName());
+ log.rollWriter();
+ MultiPut multiput = TestAsyncMultiputs.getRandomMultiPut(
+ tableNameBytes, 1000, families[0],
+ rs.getServerInfo().getServerAddress(), localutil);
+
+ try {
+ FailingSequenceFileLogWriter.setFailureMode(true);
+ rs.multiPut(multiput);
+ FailingSequenceFileLogWriter.setFailureMode(false);
+ } catch (Exception e) {
+ LOG.debug("Excepted to see an exception", e);
+ return;
+ }
+ fail();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index b7d644e..59e666a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-
+import java.util.concurrent.ExecutionException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -139,9 +139,12 @@ public class TestHLog {
* Just write multiple logs then split. Before fix for HADOOP-2283, this
* would fail.
* @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
*/
@Test
- public void testSplit() throws IOException {
+ public void testSplit() throws IOException,
+ InterruptedException, ExecutionException {
final byte [] tableName = Bytes.toBytes(TestHLog.class.getSimpleName());
final byte [] rowName = tableName;
@@ -167,7 +170,7 @@ public class TestHLog {
System.currentTimeMillis(), column));
LOG.info("Region " + i + ": " + edit);
log.append(infos[i], tableName, edit,
- System.currentTimeMillis());
+ System.currentTimeMillis()).get();
}
}
log.rollWriter();
@@ -216,7 +219,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
- wal.append(info, bytes, kvs, System.currentTimeMillis());
+ wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
}
// Now call sync and try reading. Opening a Reader before you sync just
// gives you EOFE.
@@ -234,7 +237,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
- wal.append(info, bytes, kvs, System.currentTimeMillis());
+ wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
}
reader = HLog.getReader(fs, walPath, conf);
count = 0;
@@ -253,7 +256,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), bytes, value));
- wal.append(info, bytes, kvs, System.currentTimeMillis());
+ wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
}
// Now I should have written out lots of blocks. Sync then read.
wal.sync();
@@ -350,7 +353,7 @@ public class TestHLog {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
- wal.append(regioninfo, tableName, kvs, System.currentTimeMillis());
+ wal.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
}
// Now call sync to send the data to HDFS datanodes
wal.sync(true);
@@ -448,9 +451,12 @@ public class TestHLog {
/**
* Tests that we can write out an edit, close, and then read it back in again.
* @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
*/
@Test
- public void testEditAdd() throws IOException {
+ public void testEditAdd() throws IOException,
+ InterruptedException, ExecutionException {
final int COL_COUNT = 10;
final byte [] tableName = Bytes.toBytes("tablename");
final byte [] row = Bytes.toBytes("row");
@@ -469,7 +475,7 @@ public class TestHLog {
HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName),
row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
final byte [] regionName = info.getRegionName();
- log.append(info, tableName, cols, System.currentTimeMillis());
+ log.append(info, tableName, cols, System.currentTimeMillis()).get();
long logSeqId = log.startCacheFlush(info.getRegionName());
log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion());
log.close();
@@ -541,9 +547,12 @@ public class TestHLog {
/**
* @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
*/
@Test
- public void testAppend() throws IOException {
+ public void testAppend() throws IOException,
+ InterruptedException, ExecutionException {
final int COL_COUNT = 10;
final byte [] tableName = Bytes.toBytes("tablename");
final byte [] row = Bytes.toBytes("row");
@@ -561,7 +570,7 @@ public class TestHLog {
}
HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
- log.append(hri, tableName, cols, System.currentTimeMillis());
+ log.append(hri, tableName, cols, System.currentTimeMillis()).get();
long logSeqId = log.startCacheFlush(hri.getRegionName());
log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false);
log.close();
@@ -641,13 +650,13 @@ public class TestHLog {
}
private void addEdits(HLog log, HRegionInfo hri, byte [] tableName,
- int times) throws IOException {
+ int times) throws IOException, InterruptedException, ExecutionException {
final byte [] row = Bytes.toBytes("row");
for (int i = 0; i < times; i++) {
long timestamp = System.currentTimeMillis();
WALEdit cols = new WALEdit();
cols.add(new KeyValue(row, row, row, timestamp, row));
- log.append(hri, tableName, cols, timestamp);
+ log.append(hri, tableName, cols, timestamp).checkedGet();
}
}
@@ -664,6 +673,23 @@ public class TestHLog {
}
}
+ public static class FailingSequenceFileLogWriter
+ extends SequenceFileLogWriter implements HLog.Writer {
+ static boolean enabled = false;
+ public synchronized static void setFailureMode(boolean enabled) {
+ FailingSequenceFileLogWriter.enabled = enabled;
+ }
+
+ @Override
+ public void sync() throws IOException {
+ if (FailingSequenceFileLogWriter.enabled) {
+ throw new IOException(
+ "Failing here to see if the error propogates upstream");
+ }
+ super.sync();
+ }
+ }
+
static class DumbLogEntriesVisitor implements LogEntryVisitor {
int increments = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 6a5e4a2..8d7b3b1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -52,6 +52,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -540,7 +541,8 @@ public class TestHLogSplit {
* safely commence on the master.
* */
@Test
- public void testLogRollAfterSplitStart() throws IOException {
+ public void testLogRollAfterSplitStart() throws IOException,
+ InterruptedException, ExecutionException {
// set flush interval to a large number so it doesn't interrupt us
final String F_INTERVAL = "hbase.regionserver.optionallogflushinterval";
long oldFlushInterval = conf.getLong(F_INTERVAL, 1000);
@@ -561,7 +563,7 @@ public class TestHLogSplit {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
- log.append(regioninfo, tableName, kvs, System.currentTimeMillis());
+ log.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
}
// Send the data to HDFS datanodes and close the HDFS writer
log.sync(true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
index d1f6890..2297b40 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
@@ -97,7 +97,7 @@ public class TestLogActionsListener {
KeyValue kv = new KeyValue(b,b,b);
WALEdit edit = new WALEdit();
edit.add(kv);
- hlog.append(hri, b, edit, 0);
+ hlog.append(hri, b, edit, 0).get();
if (i == 10) {
hlog.addLogActionsListerner(laterList);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 62e954f..b8571c4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -432,14 +433,14 @@ public class TestWALReplay {
long now = ee.currentTimeMillis();
edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
now, rowName));
- wal.append(hri, tableName, edit, now);
+ wal.append(hri, tableName, edit, now).get();
// Delete the c family to verify deletes make it over.
edit = new WALEdit();
now = ee.currentTimeMillis();
edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
KeyValue.Type.DeleteFamily));
- wal.append(hri, tableName, edit, now);
+ wal.append(hri, tableName, edit, now).get();
// Sync.
wal.sync();
@@ -579,7 +580,7 @@ public class TestWALReplay {
private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
final byte [] rowName, final byte [] family,
final int count, EnvironmentEdge ee, final HLog wal)
- throws IOException {
+ throws IOException, InterruptedException, ExecutionException {
String familyStr = Bytes.toString(family);
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
@@ -587,7 +588,7 @@ public class TestWALReplay {
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes,
ee.currentTimeMillis(), columnBytes));
- wal.append(hri, tableName, edit, ee.currentTimeMillis());
+ wal.append(hri, tableName, edit, ee.currentTimeMillis()).checkedGet();
}
}
[13/50] [abbrv] git commit: [master] Add stopping via RPC to
RollingRestart
Posted by ec...@apache.org.
[master] Add stopping via RPC to RollingRestart
Summary:
This diff enables giving the -S flag to RollingRestart to have the regionserver shut itself down.
This is helpful as it allows clean drain and stop without dependency on hadoopctl or any other Hstack tools
Test Plan: run on odsbaseshadow008-prn2
Reviewers: elliott, rshroff
Reviewed By: rshroff
Differential Revision: https://phabricator.fb.com/D1383296
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42878 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/301bf748
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/301bf748
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/301bf748
Branch: refs/heads/0.89-fb
Commit: 301bf748ec5e3e91e5e17febd302c59856ca294a
Parents: e14e5d5
Author: pwhite <pw...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Fri Jun 13 22:49:58 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/util/RollingRestart.java | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/301bf748/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java b/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
index a9dd753..84d7209 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
@@ -125,6 +125,10 @@ public class RollingRestart {
FAIL
}
+ void shutdownViaRPC() throws IOException {
+ admin.stopRegionServer(serverAddr, "Drained and stop via RPC by RollingRestart");
+ }
+
boolean moveRegion(final HRegionInfo region) throws Exception {
HRegionInterface destinationServer = getDestinationServer(region);
@@ -590,6 +594,8 @@ public class RollingRestart {
"Default: new temp file for drain, none for undrain");
options.addOption("readlog", "read_log_file", false,
"Reads the drain log file and prints out regions. User -location to set path to the log file. Default: false");
+ options.addOption("S", "rpcstop", false,
+ "stop the regionserver via RPC");
if (args.length == 0) {
@@ -608,6 +614,7 @@ public class RollingRestart {
int getOpFrequency = RollingRestart.DEFAULT_GETOP_FREQUENCY;
int sleepIntervalBeforeRestart = RollingRestart.DEFAULT_SLEEP_BEFORE_RESTART_INTERVAL;
boolean useHadoopCtl = true;
+ boolean stopViaRPC = false;
int port = HConstants.DEFAULT_REGIONSERVER_PORT;
RollingRestartMode mode = null;
@@ -686,6 +693,13 @@ public class RollingRestart {
port = Integer.parseInt(cmd.getOptionValue("p"));
}
+ if (cmd.hasOption("S")) {
+ if (mode != RollingRestartMode.DRAIN_AND_STOP_ONLY) {
+ LOG.error("Stop via RPC only works if drain and stop only is enabled (-o option)");
+ System.exit(1);
+ }
+ stopViaRPC = true;
+ }
RollingRestart rr;
try {
@@ -723,7 +737,11 @@ public class RollingRestart {
break;
case DRAIN_AND_STOP_ONLY:
rr.drainServer(drainedLogFile);
- rr.stop();
+ if (!stopViaRPC) {
+ rr.stop();
+ } else {
+ rr.shutdownViaRPC();
+ }
break;
case ALL:
rr.drainServer(drainedLogFile);
[03/50] [abbrv] git commit: [master] Fix generate version for svn
based scm's
Posted by ec...@apache.org.
[master] Fix generate version for svn based scm's
Summary: Git worked svn was broken. Fixed that
Test Plan: tested on a local svn checkout
Reviewers: oozie, adela
Reviewed By: adela
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1354311
Tasks: 4412611
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42559 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2c5c9ac1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2c5c9ac1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2c5c9ac1
Branch: refs/heads/0.89-fb
Commit: 2c5c9ac10ab438d2250e954195f80b4da9917502
Parents: 5425df5
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu May 29 18:10:44 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:43:00 2014 -0700
----------------------------------------------------------------------
bin/devSupport/generate_version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c5c9ac1/bin/devSupport/generate_version.py
----------------------------------------------------------------------
diff --git a/bin/devSupport/generate_version.py b/bin/devSupport/generate_version.py
index dfa7397..91fe11f 100755
--- a/bin/devSupport/generate_version.py
+++ b/bin/devSupport/generate_version.py
@@ -71,7 +71,7 @@ def get_svn_version():
"""
svn_info = check_output(["svn", "info"])
rev_line = [line.strip() for line in svn_info.splitlines(True)
- if line.strip.startswith("Last Changed Rev")][0]
+ if line.strip().startswith("Last Changed Rev")][0]
rev = rev_line.split()[-1]
return rev
[16/50] [abbrv] git commit: [master] Pull
IntegrationTestBigLinkedList from upstream 0.96
Posted by ec...@apache.org.
[master] Pull IntegrationTestBigLinkedList from upstream 0.96
Summary: IntegrationTestBigLinkedList has been the best test from open source's suite of integration tests. So I a pulling it first. This is a pretty direct port; however I did change it so that tables are created pre-split.
Test Plan: Testing this on dev038
Reviewers: daviddeng, manukranthk
Reviewed By: manukranthk
Subscribers: adela, hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1365156
Tasks: 4185690
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42798 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8486856
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8486856
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8486856
Branch: refs/heads/0.89-fb
Commit: b848685610a2662054a741649f11aff1ba4f606f
Parents: df1f91f
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 11 17:10:41 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../test/IntegrationTestBigLinkedList.java | 1116 ++++++++++++++++++
1 file changed, 1116 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8486856/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
new file mode 100644
index 0000000..38230a1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -0,0 +1,1116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.test;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.IntegrationTestBase;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.collect.Sets;
+
+/**
+ * This is an integration test borrowed from goraci, written by Keith Turner,
+ * which is in turn inspired by the Accumulo test called continous ingest (ci).
+ * The original source code can be found here:
+ * https://github.com/keith-turner/goraci
+ * https://github.com/enis/goraci/
+ *
+ * Apache Accumulo [0] has a simple test suite that verifies that data is not
+ * lost at scale. This test suite is called continuous ingest. This test runs
+ * many ingest clients that continually create linked lists containing 25
+ * million nodes. At some point the clients are stopped and a map reduce job is
+ * run to ensure no linked list has a hole. A hole indicates data was lost.??
+ *
+ * The nodes in the linked list are random. This causes each linked list to
+ * spread across the table. Therefore if one part of a table loses data, then it
+ * will be detected by references in another part of the table.
+ *
+ * THE ANATOMY OF THE TEST
+ *
+ * Below is rough sketch of how data is written. For specific details look at
+ * the Generator code.
+ *
+ * 1 Write out 1 million nodes? 2 Flush the client? 3 Write out 1 million that
+ * reference previous million? 4 If this is the 25th set of 1 million nodes,
+ * then update 1st set of million to point to last? 5 goto 1
+ *
+ * The key is that nodes only reference flushed nodes. Therefore a node should
+ * never reference a missing node, even if the ingest client is killed at any
+ * point in time.
+ *
+ * When running this test suite w/ Accumulo there is a script running in
+ * parallel called the Aggitator that randomly and continuously kills server
+ * processes.?? The outcome was that many data loss bugs were found in Accumulo
+ * by doing this.? This test suite can also help find bugs that impact uptime
+ * and stability when? run for days or weeks.??
+ *
+ * This test suite consists the following? - a few Java programs? - a little
+ * helper script to run the java programs - a maven script to build it.??
+ *
+ * When generating data, its best to have each map task generate a multiple of
+ * 25 million. The reason for this is that circular linked list are generated
+ * every 25M. Not generating a multiple in 25M will result in some nodes in the
+ * linked list not having references. The loss of an unreferenced node can not
+ * be detected.
+ *
+ *
+ * Below is a description of the Java programs
+ *
+ * Generator - A map only job that generates data. As stated previously,?
+ * its best to generate data in multiples of 25M.
+ *
+ * Verify - A map reduce job that looks for holes. Look at the counts after running. REFERENCED and
+ * UNREFERENCED are? ok, any UNDEFINED counts are bad. Do not run at the? same
+ * time as the Generator.
+ *
+ * Walker - A standalone program that start following a linked list? and emits timing info.??
+ *
+ * Print - A standalone program that prints nodes in the linked list
+ *
+ * Delete - A standalone program that deletes a single node
+ *
+ * This class can be run as a unit test, as an integration test, or from the command line
+ */
+public class IntegrationTestBigLinkedList extends IntegrationTestBase {
+ private static final byte[] NO_KEY = new byte[1];
+
+ protected static String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
+
+ protected static String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList";
+
+ private static byte[] FAMILY_NAME = Bytes.toBytes("meta");
+
+ //link to the id of the prev node in the linked list
+ private static final byte[] COLUMN_PREV = Bytes.toBytes("prev");
+
+ //identifier of the mapred task that generated this row
+ private static final byte[] COLUMN_CLIENT = Bytes.toBytes("client");
+
+ //the id of the row within the same client.
+ private static final byte[] COLUMN_COUNT = Bytes.toBytes("count");
+
+ /** How many rows to write per map task. This has to be a multiple of 25M */
+ private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY
+ = "IntegrationTestBigLinkedList.generator.num_rows";
+
+ private static final String GENERATOR_NUM_MAPPERS_KEY
+ = "IntegrationTestBigLinkedList.generator.map.tasks";
+
+ private static final String GENERATOR_WIDTH_KEY
+ = "IntegrationTestBigLinkedList.generator.width";
+
+ private static final String GENERATOR_WRAP_KEY
+ = "IntegrationTestBigLinkedList.generator.wrap";
+
+
+ private static final int MISSING_ROWS_TO_LOG = 50;
+
+ private static final int WIDTH_DEFAULT = 1000000;
+ private static final int WRAP_DEFAULT = 25;
+ private static final int ROWKEY_LENGTH = 16;
+
+ private String toRun;
+ private String[] otherArgs;
+
+ static class CINode {
+ byte[] key;
+ byte[] prev;
+ String client;
+ long count;
+ }
+
+ /**
+ * A Map only job that generates random linked list and stores them.
+ */
+ static class Generator extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Generator.class);
+
+ static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
+ static class GeneratorInputSplit extends InputSplit implements Writable {
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 1;
+ }
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ }
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ }
+ }
+
+ static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
+ private long count;
+ private long numNodes;
+ private Random rand;
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public BytesWritable getCurrentKey() throws IOException, InterruptedException {
+ byte[] bytes = new byte[ROWKEY_LENGTH];
+ rand.nextBytes(bytes);
+ return new BytesWritable(bytes);
+ }
+
+ @Override
+ public NullWritable getCurrentValue() throws IOException, InterruptedException {
+ return NullWritable.get();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return (float)(count / (double)numNodes);
+ }
+
+ @Override
+ public void initialize(InputSplit arg0, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
+ rand = new Random();
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ return count++ < numNodes;
+ }
+
+ }
+
+ @Override
+ public RecordReader<BytesWritable,NullWritable> createRecordReader(
+ InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
+ GeneratorRecordReader rr = new GeneratorRecordReader();
+ rr.initialize(split, context);
+ return rr;
+ }
+
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
+ int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1);
+
+ ArrayList<InputSplit> splits = new ArrayList<>(numMappers);
+
+ for (int i = 0; i < numMappers; i++) {
+ splits.add(new GeneratorInputSplit());
+ }
+
+ return splits;
+ }
+ }
+
+ /** Ensure output files from prev-job go to map inputs for current job */
+ static class OneFilePerMapperSFIF<K, V> extends SequenceFileInputFormat<K, V> {
+ @Override
+ protected boolean isSplitable(JobContext context, Path filename) {
+ return false;
+ }
+ }
+
+ /**
+ * Some ASCII art time:
+ * [ . . . ] represents one batch of random longs of length WIDTH
+ *
+ * _________________________
+ * | ______ |
+ * | | ||
+ * __+_________________+_____ ||
+ * v v v |||
+ * first = [ . . . . . . . . . . . ] |||
+ * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
+ * | | | | | | | | | | | |||
+ * prev = [ . . . . . . . . . . . ] |||
+ * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
+ * | | | | | | | | | | | |||
+ * current = [ . . . . . . . . . . . ] |||
+ * |||
+ * ... |||
+ * |||
+ * last = [ . . . . . . . . . . . ] |||
+ * | | | | | | | | | | |-----|||
+ * | |--------||
+ * |___________________________|
+ */
+ static class GeneratorMapper
+ extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
+
+ byte[][] first = null;
+ byte[][] prev = null;
+ byte[][] current = null;
+ byte[] id;
+ long count = 0;
+ int curIteration = 0;
+ HTable table;
+ long numNodes;
+ long wrap;
+ int width;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ id = Bytes.toBytes("Job: "+context.getJobID() + " Task: " + context.getTaskAttemptID());
+ Configuration conf = context.getConfiguration();
+ table = new HTable(conf, getTableName(conf));
+ table.setAutoFlush(false, true);
+ table.setWriteBufferSize(4 * 1024 * 1024);
+ this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
+ current = new byte[this.width][];
+ int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
+ this.wrap = (long)wrapMultiplier * width;
+ this.numNodes = context.getConfiguration().getLong(
+ GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
+ if (this.numNodes < this.wrap) {
+ this.wrap = this.numNodes;
+ }
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException ,InterruptedException {
+ table.close();
+ }
+
+ @Override
+ protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
+ current[curIteration] = new byte[key.getLength()];
+ System.arraycopy(key.getBytes(), 0, current[curIteration], 0, key.getLength());
+ if (++curIteration == current.length) {
+ persist(output, count, prev, current, id);
+ curIteration = 0;
+
+ if (first == null)
+ first = current;
+ prev = current;
+ current = new byte[this.width][];
+
+ count += current.length;
+ output.setStatus("Count " + count);
+
+ if (count % wrap == 0) {
+ // this block of code turns the 1 million linked list of length 25 into one giant
+ //circular linked list of 25 million
+ circularLeftShift(first);
+
+ persist(output, -1, prev, first, null);
+
+ first = null;
+ prev = null;
+ }
+ }
+ }
+
+ private static <T> void circularLeftShift(T[] first) {
+ T ez = first[0];
+ System.arraycopy(first, 1, first, 0, first.length - 1);
+ first[first.length - 1] = ez;
+ }
+
+ private void persist(Context output, long count, byte[][] prev, byte[][] current, byte[] id)
+ throws IOException {
+ for (int i = 0; i < current.length; i++) {
+ Put put = new Put(current[i]);
+ put.add(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]);
+
+ if (count >= 0) {
+ put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
+ }
+ if (id != null) {
+ put.add(FAMILY_NAME, COLUMN_CLIENT, id);
+ }
+ table.put(put);
+
+ if (i % 1000 == 0) {
+ // Tickle progress every so often else maprunner will think us hung
+ output.progress();
+ }
+ }
+
+ table.flushCommits();
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.out.println("Usage : " + Generator.class.getSimpleName() +
+ " <num mappers> <num nodes per map> <tmp output dir> [<width> <wrap multiplier>]");
+ System.out.println(" where <num nodes per map> should be a multiple of " +
+ " width*wrap multiplier, 25M by default");
+ return 0;
+ }
+
+ int numMappers = Integer.parseInt(args[0]);
+ long numNodes = Long.parseLong(args[1]);
+ Path tmpOutput = new Path(args[2]);
+ Integer width = (args.length < 4) ? null : Integer.parseInt(args[3]);
+ Integer wrapMuplitplier = (args.length < 5) ? null : Integer.parseInt(args[4]);
+ return run(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
+ }
+
+ protected void createSchema(int shards) throws IOException {
+ HBaseAdmin admin = new HBaseAdmin(getConf());
+ StringBytes tableName = getTableName(getConf());
+ if (!admin.tableExists(tableName)) {
+ // TODO:(elliott) Add StringBytes constructor for HTD.
+ HTableDescriptor htd = new HTableDescriptor(getTableName(getConf()).getBytes());
+ htd.addFamily(new HColumnDescriptor(FAMILY_NAME));
+ byte[] endRk = new byte[ROWKEY_LENGTH];
+
+ for (int i = 0; i < endRk.length; i++) {
+ endRk[i] = Byte.MAX_VALUE;
+ }
+ admin.createTable(htd, new byte[ROWKEY_LENGTH], endRk, shards);
+ }
+ admin.close();
+ }
+
+ public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
+ LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
+ + ", numNodes=" + numNodes);
+ Job job = new Job(getConf());
+
+ job.setJobName("Random Input Generator");
+ job.setNumReduceTasks(0);
+ job.setJarByClass(getClass());
+
+ job.setInputFormatClass(GeneratorInputFormat.class);
+ job.setOutputKeyClass(BytesWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
+
+ job.setMapperClass(Mapper.class); //identity mapper
+
+ FileOutputFormat.setOutputPath(job, tmpOutput);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ public int runGenerator(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
+ LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
+ createSchema(numMappers * 2);
+
+ Job job = new Job(getConf());
+
+ job.setJobName("Link Generator");
+ job.setNumReduceTasks(0);
+ job.setJarByClass(getClass());
+
+ FileInputFormat.setInputPaths(job, tmpOutput);
+ job.setInputFormatClass(OneFilePerMapperSFIF.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
+
+ job.setMapperClass(GeneratorMapper.class);
+
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+ TableMapReduceUtil.addDependencyJars(job);
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ public int run(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
+ int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
+ if (ret > 0) {
+ return ret;
+ }
+ return runGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
+ }
+ }
+
+ /**
+ * A Map Reduce job that verifies that the linked lists generated by
+ * {@link Generator} do not have any holes.
+ */
+ static class Verify extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Verify.class);
+ private static final BytesWritable DEF = new BytesWritable(NO_KEY);
+
+ private Job job;
+
+ public static class VerifyMapper extends TableMapper<BytesWritable, BytesWritable> {
+ private BytesWritable row = new BytesWritable();
+ private BytesWritable ref = new BytesWritable();
+
+ @Override
+ protected void map(ImmutableBytesWritable key, Result value, Context context)
+ throws IOException ,InterruptedException {
+ byte[] rowKey = key.get();
+ row.set(rowKey, 0, rowKey.length);
+ context.write(row, DEF);
+ byte[] prev = value.getValue(FAMILY_NAME, COLUMN_PREV);
+ if (prev != null && prev.length > 0) {
+ ref.set(prev, 0, prev.length);
+ context.write(ref, row);
+ } else {
+ LOG.warn(String.format("Prev is not set for: %s", Bytes.toStringBinary(rowKey)));
+ }
+ }
+ }
+
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT, EXTRAREFERENCES
+ }
+
+ public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
+ private ArrayList<byte[]> refs = new ArrayList<>();
+
+ private AtomicInteger rows = new AtomicInteger(0);
+
+ @Override
+ public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
+ throws IOException, InterruptedException {
+
+ int defCount = 0;
+
+ refs.clear();
+ for (BytesWritable type : values) {
+ if (type.getLength() == DEF.getLength()) {
+ defCount++;
+ } else {
+ byte[] bytes = new byte[type.getLength()];
+ System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
+ refs.add(bytes);
+ }
+ }
+
+ // TODO check for more than one def, should not happen
+
+ StringBuilder refsSb = null;
+ String keyString = null;
+ if (defCount == 0 || refs.size() != 1) {
+ refsSb = new StringBuilder();
+ String comma = "";
+ for (byte[] ref : refs) {
+ refsSb.append(comma);
+ comma = ",";
+ refsSb.append(Bytes.toStringBinary(ref));
+ }
+ keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength());
+
+ LOG.error("Linked List error: Key = " + keyString + " References = " + refsSb.toString());
+ }
+
+ if (defCount == 0 && refs.size() > 0) {
+ // this is bad, found a node that is referenced but not defined. It must have been
+ // lost, emit some info about this node for debugging purposes.
+ context.write(new Text(keyString), new Text(refsSb.toString()));
+ context.getCounter(Counts.UNDEFINED).increment(1);
+ if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
+ context.getCounter("undef", keyString).increment(1);
+ }
+ } else if (defCount > 0 && refs.size() == 0) {
+ // node is defined but not referenced
+ context.write(new Text(keyString), new Text("none"));
+ context.getCounter(Counts.UNREFERENCED).increment(1);
+ if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
+ context.getCounter("unref", keyString).increment(1);
+ }
+ } else {
+ if (refs.size() > 1) {
+ if (refsSb != null) {
+ context.write(new Text(keyString), new Text(refsSb.toString()));
+ }
+ context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
+ }
+ // node is defined and referenced
+ context.getCounter(Counts.REFERENCED).increment(1);
+ }
+
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 2) {
+ System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>");
+ return 0;
+ }
+
+ String outputDir = args[0];
+ int numReducers = Integer.parseInt(args[1]);
+
+ return run(outputDir, numReducers);
+ }
+
+ public int run(String outputDir, int numReducers) throws Exception {
+ return run(new Path(outputDir), numReducers);
+ }
+
+ public int run(Path outputDir, int numReducers) throws Exception {
+ LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
+
+ job = new Job(getConf());
+
+ job.setJobName("Link Verifier");
+ job.setNumReduceTasks(numReducers);
+ job.setJarByClass(getClass());
+
+ Scan scan = new Scan();
+ scan.addColumn(FAMILY_NAME, COLUMN_PREV);
+ scan.setCaching(10000);
+ scan.setCacheBlocks(false);
+
+ TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getString(), scan,
+ VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ job.setReducerClass(VerifyReducer.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, outputDir);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ @SuppressWarnings("deprecation")
+ public boolean verify(long expectedReferenced) throws Exception {
+ if (job == null) {
+ throw new IllegalStateException("You should call run() first");
+ }
+
+ Counters counters = job.getCounters();
+
+ Counter referenced = counters.findCounter(Counts.REFERENCED);
+ Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
+ Counter undefined = counters.findCounter(Counts.UNDEFINED);
+ Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
+
+ boolean success = true;
+ //assert
+ if (expectedReferenced != referenced.getValue()) {
+ LOG.error("Expected referenced count does not match with actual referenced count. " +
+ "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
+ success = false;
+ }
+
+ if (unreferenced.getValue() > 0) {
+ boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
+ LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
+ + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
+ success = false;
+ }
+
+ if (undefined.getValue() > 0) {
+ LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
+ success = false;
+ }
+
+ if (!success) {
+ Configuration conf = job.getConfiguration();
+ HConnection conn = HConnectionManager.getConnection(conf);
+ StringBytes tableName = getTableName(conf);
+ CounterGroup g = counters.getGroup("undef");
+ Iterator<Counter> it = g.iterator();
+ while (it.hasNext()) {
+ String keyString = it.next().getName();
+ byte[] key = Bytes.toBytes(keyString);
+ HRegionLocation loc = conn.relocateRegion(tableName, key);
+ LOG.error("undefined row " + keyString + ", " + loc);
+ }
+ g = counters.getGroup("unref");
+ it = g.iterator();
+ while (it.hasNext()) {
+ String keyString = it.next().getName();
+ byte[] key = Bytes.toBytes(keyString);
+ HRegionLocation loc = conn.relocateRegion(tableName, key);
+ LOG.error("unreferred row " + keyString + ", " + loc);
+ }
+ }
+ return success;
+ }
+ }
+
+ /**
+ * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration
+ * adds more data.
+ */
+ static class Loop extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Loop.class);
+
+ IntegrationTestBigLinkedList it;
+
+ protected void runGenerator(int numMappers, long numNodes,
+ String outputDir, Integer width, Integer wrapMuplitplier) throws Exception {
+ Path outputPath = new Path(outputDir);
+ UUID uuid = UUID.randomUUID(); //create a random UUID.
+ Path generatorOutput = new Path(outputPath, uuid.toString());
+
+ Generator generator = new Generator();
+ generator.setConf(getConf());
+ int retCode = generator.run(numMappers, numNodes, generatorOutput, width, wrapMuplitplier);
+ if (retCode > 0) {
+ throw new RuntimeException("Generator failed with return code: " + retCode);
+ }
+ }
+
+ protected void runVerify(String outputDir,
+ int numReducers, long expectedNumNodes) throws Exception {
+ Path outputPath = new Path(outputDir);
+ UUID uuid = UUID.randomUUID(); //create a random UUID.
+ Path iterationOutput = new Path(outputPath, uuid.toString());
+
+ Verify verify = new Verify();
+ verify.setConf(getConf());
+ int retCode = verify.run(iterationOutput, numReducers);
+ if (retCode > 0) {
+ throw new RuntimeException("Verify.run failed with return code: " + retCode);
+ }
+
+ if (!verify.verify(expectedNumNodes)) {
+ throw new RuntimeException("Verify.verify failed");
+ }
+
+ LOG.info("Verify finished with succees. Total nodes=" + expectedNumNodes);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 5) {
+ System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers> [<width> <wrap multiplier>]");
+ return 1;
+ }
+ LOG.info("Running Loop with args:" + Arrays.deepToString(args));
+
+ int numIterations = Integer.parseInt(args[0]);
+ int numMappers = Integer.parseInt(args[1]);
+ long numNodes = Long.parseLong(args[2]);
+ String outputDir = args[3];
+ int numReducers = Integer.parseInt(args[4]);
+ Integer width = (args.length < 6) ? null : Integer.parseInt(args[5]);
+ Integer wrapMuplitplier = (args.length < 7) ? null : Integer.parseInt(args[6]);
+
+ long expectedNumNodes = 0;
+
+ if (numIterations < 0) {
+ numIterations = Integer.MAX_VALUE; //run indefinitely (kind of)
+ }
+
+ for (int i = 0; i < numIterations; i++) {
+ LOG.info("Starting iteration = " + i);
+ runGenerator(numMappers, numNodes, outputDir, width, wrapMuplitplier);
+ expectedNumNodes += numMappers * numNodes;
+
+ runVerify(outputDir, numReducers, expectedNumNodes);
+ }
+
+ return 0;
+ }
+ }
+
+ /**
+ * A stand alone program that prints out portions of a list created by {@link Generator}
+ */
+ private static class Print extends Configured implements Tool {
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+ options.addOption("s", "start", true, "start key");
+ options.addOption("e", "end", true, "end key");
+ options.addOption("l", "limit", true, "number to print");
+
+ GnuParser parser = new GnuParser();
+ CommandLine cmd = null;
+ try {
+ cmd = parser.parse(options, args);
+ if (cmd.getArgs().length != 0) {
+ throw new ParseException("Command takes no arguments");
+ }
+ } catch (ParseException e) {
+ System.err.println("Failed to parse command line " + e.getMessage());
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(getClass().getSimpleName(), options);
+ System.exit(-1);
+ }
+
+ HTable table = new HTable(getConf(), getTableName(getConf()));
+
+ Scan scan = new Scan();
+ scan.setBatch(10000);
+
+ if (cmd.hasOption("s"))
+ scan.setStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));
+
+ if (cmd.hasOption("e"))
+ scan.setStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
+
+ int limit;
+ if (cmd.hasOption("l"))
+ limit = Integer.parseInt(cmd.getOptionValue("l"));
+ else
+ limit = 100;
+
+ ResultScanner scanner = table.getScanner(scan);
+
+ CINode node = new CINode();
+ Result result = scanner.next();
+ int count = 0;
+ while (result != null && count++ < limit) {
+ node = getCINode(result, node);
+ System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
+ Bytes.toStringBinary(node.prev), node.count, node.client);
+ result = scanner.next();
+ }
+ scanner.close();
+ table.close();
+
+ return 0;
+ }
+ }
+
+ /**
+ * A stand alone program that deletes a single node.
+ */
+ private static class Delete extends Configured implements Tool {
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length != 1) {
+ System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
+ return 0;
+ }
+ byte[] val = Bytes.toBytesBinary(args[0]);
+
+ org.apache.hadoop.hbase.client.Delete delete
+ = new org.apache.hadoop.hbase.client.Delete(val);
+
+ HTable table = new HTable(getConf(), getTableName(getConf()));
+
+ table.delete(delete);
+ table.flushCommits();
+ table.close();
+
+ System.out.println("Delete successful");
+ return 0;
+ }
+ }
+
+ /**
+ * A stand alone program that follows a linked list created by {@link Generator} and prints timing info.
+ */
+ private static class Walker extends Configured implements Tool {
+ @Override
+ public int run(String[] args) throws IOException {
+ Options options = new Options();
+ options.addOption("n", "num", true, "number of queries");
+ options.addOption("s", "start", true, "key to start at, binary string");
+ options.addOption("l", "logevery", true, "log every N queries");
+
+ GnuParser parser = new GnuParser();
+ CommandLine cmd = null;
+ try {
+ cmd = parser.parse(options, args);
+ if (cmd.getArgs().length != 0) {
+ throw new ParseException("Command takes no arguments");
+ }
+ } catch (ParseException e) {
+ System.err.println("Failed to parse command line " + e.getMessage());
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(getClass().getSimpleName(), options);
+ System.exit(-1);
+ }
+
+ long maxQueries = Long.MAX_VALUE;
+ if (cmd.hasOption('n')) {
+ maxQueries = Long.parseLong(cmd.getOptionValue("n"));
+ }
+ Random rand = new Random();
+ boolean isSpecificStart = cmd.hasOption('s');
+ byte[] startKey = isSpecificStart ? Bytes.toBytesBinary(cmd.getOptionValue('s')) : null;
+ int logEvery = cmd.hasOption('l') ? Integer.parseInt(cmd.getOptionValue('l')) : 1;
+
+ HTable table = new HTable(getConf(), getTableName(getConf()));
+ long numQueries = 0;
+ // If isSpecificStart is set, only walk one list from that particular node.
+ // Note that in case of circular (or P-shaped) list it will walk forever, as is
+ // the case in normal run without startKey.
+ while (numQueries < maxQueries && (numQueries == 0 || !isSpecificStart)) {
+ if (!isSpecificStart) {
+ startKey = new byte[ROWKEY_LENGTH];
+ rand.nextBytes(startKey);
+ }
+ CINode node = findStartNode(table, startKey);
+ if (node == null && isSpecificStart) {
+ System.err.printf("Start node not found: %s \n", Bytes.toStringBinary(startKey));
+ }
+ numQueries++;
+ while (node != null && node.prev.length != NO_KEY.length && numQueries < maxQueries) {
+ byte[] prev = node.prev;
+ long t1 = System.currentTimeMillis();
+ node = getNode(prev, table, node);
+ long t2 = System.currentTimeMillis();
+ if (numQueries % logEvery == 0) {
+ System.out.printf("CQ %d: %d %s \n", numQueries, t2 - t1, Bytes.toStringBinary(prev));
+ }
+ numQueries++;
+ if (node == null) {
+ System.err.printf("UNDEFINED NODE %s \n", Bytes.toStringBinary(prev));
+ } else if (node.prev.length == NO_KEY.length) {
+ System.err.printf("TERMINATING NODE %s \n", Bytes.toStringBinary(node.key));
+ }
+ }
+ }
+
+ table.close();
+ return 0;
+ }
+
+ private static CINode findStartNode(HTable table, byte[] startKey) throws IOException {
+ Scan scan = new Scan();
+ scan.setStartRow(startKey);
+ scan.setBatch(1);
+ scan.addColumn(FAMILY_NAME, COLUMN_PREV);
+
+ long t1 = System.currentTimeMillis();
+ ResultScanner scanner = table.getScanner(scan);
+ Result result = scanner.next();
+ long t2 = System.currentTimeMillis();
+ scanner.close();
+
+ if ( result != null) {
+ CINode node = getCINode(result, new CINode());
+ System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key));
+ return node;
+ }
+
+ System.out.println("FSR " + (t2 - t1));
+
+ return null;
+ }
+
+ private CINode getNode(byte[] row, HTable table, CINode node) throws IOException {
+ Get get = new Get.Builder(row).addColumn(FAMILY_NAME, COLUMN_PREV).create();
+ Result result = table.get(get);
+ return getCINode(result, node);
+ }
+ }
+
+ static StringBytes getTableName(Configuration conf) {
+ return new StringBytes(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME));
+ }
+
+ private static CINode getCINode(Result result, CINode node) {
+ node.key = Bytes.copyOfByteArray(result.getRow());
+ if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
+ node.prev = Bytes.copyOfByteArray(result.getValue(FAMILY_NAME, COLUMN_PREV));
+ } else {
+ node.prev = NO_KEY;
+ }
+ if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
+ node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
+ } else {
+ node.count = -1;
+ }
+ if (result.containsColumn(FAMILY_NAME, COLUMN_CLIENT)) {
+ node.client = Bytes.toString(result.getValue(FAMILY_NAME, COLUMN_CLIENT));
+ } else {
+ node.client = "";
+ }
+ return node;
+ }
+
+ protected IntegrationTestingUtility util;
+
+ private void usage() {
+ System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]");
+ System.err.println(" where COMMAND is one of:");
+ System.err.println("");
+ System.err.println(" Generator A map only job that generates data.");
+ System.err.println(" Verify A map reduce job that looks for holes");
+ System.err.println(" Look at the counts after running");
+ System.err.println(" REFERENCED and UNREFERENCED are ok");
+ System.err.println(" any UNDEFINED counts are bad. Do not");
+ System.err.println(" run at the same time as the Generator.");
+ System.err.println(" Walker A standalong program that starts ");
+ System.err.println(" following a linked list and emits");
+ System.err.println(" timing info.");
+ System.err.println(" Print A standalone program that prints nodes");
+ System.err.println(" in the linked list.");
+ System.err.println(" Delete A standalone program that deletes a?");
+ System.err.println(" single node.");
+ System.err.println(" Loop A program to Loop through Generator and");
+ System.err.println(" Verify steps");
+ System.err.println("\t ");
+ System.err.flush();
+ }
+
+ @Override
+ protected void processOptions(CommandLine cmd) {
+ super.processOptions(cmd);
+ String[] args = cmd.getArgs();
+ //get the class, run with the conf
+ if (args.length < 1) {
+ printUsage();
+ throw new RuntimeException("Incorrect Number of args.");
+ }
+ toRun = args[0];
+ otherArgs = Arrays.copyOfRange(args, 1, args.length);
+ }
+
+ @Override public void setUpCluster() throws Exception {
+
+ }
+
+ @Override
+ public int runTestFromCommandLine() throws Exception {
+
+ Tool tool;
+ switch (toRun) {
+ case "Generator":
+ tool = new Generator();
+ break;
+ case "Verify":
+ tool = new Verify();
+ break;
+ case "Loop":
+ Loop loop = new Loop();
+ loop.it = this;
+ tool = loop;
+ break;
+ case "Walker":
+ tool = new Walker();
+ break;
+ case "Print":
+ tool = new Print();
+ break;
+ case "Delete":
+ tool = new Delete();
+ break;
+ default:
+ usage();
+ throw new RuntimeException("Unknown arg");
+ }
+
+ return ToolRunner.run(getConf(), tool, otherArgs);
+ }
+
+ @Override
+ public String getTablename() {
+ Configuration c = getConf();
+ return c.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME);
+ }
+
+ @Override
+ protected Set<String> getColumnFamilies() {
+ return Sets.newHashSet(Bytes.toString(FAMILY_NAME));
+ }
+
+ private static void setJobConf(Job job, int numMappers, long numNodes,
+ Integer width, Integer wrapMultiplier) {
+ job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+ job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+ if (width != null) {
+ job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width);
+ }
+ if (wrapMultiplier != null) {
+ job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMultiplier);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ new IntegrationTestBigLinkedList().doStaticMain(args);
+ }
+}
[08/50] [abbrv] git commit: [HBASE-11336] Escape regionName and
start/endKeys for HTML. Show numbers with thousand separators and right align
Posted by ec...@apache.org.
[HBASE-11336] Escape regionName and start/endKeys for HTML. Show numbers with thousand separators and right align
Summary: As title.
Test Plan: `TestFsUtils`
Reviewers: manukranthk, gauravm, fan, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1383779
Tasks: 4513820
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42885 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/31b1af17
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/31b1af17
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/31b1af17
Branch: refs/heads/0.89-fb
Commit: 31b1af177320feb78a9ba70d126f9bbe416f51ca
Parents: 301bf74
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Sun Jun 15 13:14:25 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
src/main/resources/hbase-webapps/master/table.jsp | 9 +++++----
src/main/resources/hbase-webapps/static/hbase.css | 4 ++++
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/31b1af17/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 65bfa49..704e43e 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -13,6 +13,7 @@
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.util.StringUtils"
+ import="org.apache.commons.lang.StringEscapeUtils"
import="java.util.Map"
import="org.apache.hadoop.hbase.HConstants"%><%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
@@ -153,11 +154,11 @@
"http://" + hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort + "/";
%>
<tr>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getRegionName())%></td>
+ <td><%= StringEscapeUtils.escapeHtml(Bytes.toStringBinary(hriEntry.getKey().getRegionName())) %></td>
<td><a href="<%= urlRegionServer %>"><%= hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort %></a></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getStartKey())%></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getEndKey())%></td>
- <td><%= StringUtils.byteDesc(regionSizes.get(hriEntry.getKey())) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(Bytes.toStringBinary(hriEntry.getKey().getStartKey())) %></td>
+ <td><%= StringEscapeUtils.escapeHtml(Bytes.toStringBinary(hriEntry.getKey().getEndKey())) %></td>
+ <td class="number"><%= String.format("%,d", regionSizes.get(hriEntry.getKey())) %></td>
</tr>
<% } %>
</table>
http://git-wip-us.apache.org/repos/asf/hbase/blob/31b1af17/src/main/resources/hbase-webapps/static/hbase.css
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/static/hbase.css b/src/main/resources/hbase-webapps/static/hbase.css
index 4bc5301..8f2063a 100644
--- a/src/main/resources/hbase-webapps/static/hbase.css
+++ b/src/main/resources/hbase-webapps/static/hbase.css
@@ -31,3 +31,7 @@ tr.task-monitor-IDLE td {
tr.task-monitor-ABORTED td {
background-color: #f66;
}
+
+td.number {
+ text-align: right;
+}
[12/50] [abbrv] git commit: [HBASE-11254] Improve retry logic in
HTableMultiplexer.HTableFlushWorker
Posted by ec...@apache.org.
[HBASE-11254] Improve retry logic in HTableMultiplexer.HTableFlushWorker
Summary:
- Refactored code a little bit to avoid duplication
- Improved retry logic in HtableMultiplexer: every failed put is scheduled over executor for retrying. Every time the put is readded, we increase the time when it is going to be executed. Additionally I introduce a counter, which counts this kind of puts and limits them, so that we don't run into OOM (this parameter is also client configurable parameter).
Test Plan: ran TestHTableMultplexer test
Reviewers: liyintang, aaiyer, rshroff, manukranthk, fan, gauravm, daviddeng, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1348746
Tasks: 4245275
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42672 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3183638
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3183638
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3183638
Branch: refs/heads/0.89-fb
Commit: d318363840c236d94fcc5a26ef22b268fe8aec5d
Parents: a404f16
Author: adela <ad...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 4 22:11:05 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/client/HTableMultiplexer.java | 163 ++++++++++++-------
1 file changed, 101 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3183638/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
index e8d3ad3..9156fe4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -41,6 +42,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
@@ -66,21 +68,23 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
public class HTableMultiplexer {
private static final Log LOG = LogFactory.getLog(HTableMultiplexer.class.getName());
- private Map<byte[], HTable> tableNameToHTableMap;
+ private final Map<byte[], HTable> tableNameToHTableMap;
/** The map between each region server to its corresponding buffer queue */
- private Map<HServerAddress, LinkedBlockingQueue<PutStatus>>
+ private final Map<HServerAddress, LinkedBlockingQueue<PutStatus>>
serverToBufferQueueMap;
/** The map between each region server to its flush worker */
- private Map<HServerAddress, HTableFlushWorker> serverToFlushWorkerMap;
-
- private Configuration conf;
- private HConnection connection;
- private int retryNum;
- private int perRegionServerBufferQueueSize;
- private ScheduledExecutorService executor;
- private long frequency = 100;
+ private final Map<HServerAddress, HTableFlushWorker> serverToFlushWorkerMap;
+
+ private final Configuration conf;
+ private final HConnection connection;
+ private final int retryNum;
+ // limit of retried puts scheduled for retry
+ private final int retriedInQueueMax;
+ private final int perRegionServerBufferQueueSize;
+ private final ScheduledExecutorService executor;
+ private final long frequency;
//initial number of threads in the pool
public static final int INITIAL_NUM_THREADS = 10;
@@ -99,6 +103,7 @@ public class HTableMultiplexer {
this.tableNameToHTableMap = new ConcurrentSkipListMap<byte[], HTable>(
Bytes.BYTES_COMPARATOR);
this.retryNum = conf.getInt("hbase.client.retries.number", 10);
+ this.retriedInQueueMax = conf.getInt("hbase.client.retried.inQueue", 10000);
this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize;
this.frequency = conf.getLong("hbase.htablemultiplexer.flush.frequency.ms",
100);
@@ -230,10 +235,9 @@ public class HTableMultiplexer {
serverToBufferQueueMap.put(addr, queue);
// Create the flush worker
- HTableFlushWorker worker = new HTableFlushWorker(conf, addr,
- this.connection, this, queue);
+ HTableFlushWorker worker = new HTableFlushWorker(addr, queue);
this.serverToFlushWorkerMap.put(addr, worker);
- executor.scheduleAtFixedRate(worker, frequency, frequency, TimeUnit.MICROSECONDS);
+ executor.scheduleAtFixedRate(worker, frequency, frequency, TimeUnit.MILLISECONDS);
}
return queue;
}
@@ -559,29 +563,23 @@ public class HTableMultiplexer {
}
}
- private static class HTableFlushWorker implements Runnable {
- private HServerAddress addr;
- private Configuration conf;
- private LinkedBlockingQueue<PutStatus> queue;
- private HConnection connection;
- private HTableMultiplexer htableMultiplexer;
- private AtomicLong totalFailedPutCount;
- private AtomicLong totalSucceededPutCount;
- private AtomicLong totalRetriedPutCount;
- private AtomicInteger currentProcessingPutCount;
- private AtomicInteger minProcessingPutCount;
- private AtomicInteger maxProcessingPutCount;
- private AtomicAverageCounter avgProcessingPutCount;
- private AtomicAverageCounter averageLatency;
- private AtomicLong maxLatency;
-
- public HTableFlushWorker(Configuration conf, HServerAddress addr,
- HConnection connection, HTableMultiplexer htableMultiplexer,
- LinkedBlockingQueue<PutStatus> queue) {
+ private class HTableFlushWorker implements Runnable {
+ private final HServerAddress addr;
+ private final LinkedBlockingQueue<PutStatus> queue;
+ private final AtomicLong totalFailedPutCount;
+ private final AtomicLong totalSucceededPutCount;
+ private final AtomicLong totalRetriedPutCount;
+ private final AtomicInteger currentProcessingPutCount;
+ private final AtomicInteger minProcessingPutCount;
+ private final AtomicInteger maxProcessingPutCount;
+ private final AtomicAverageCounter avgProcessingPutCount;
+ private final AtomicAverageCounter averageLatency;
+ private final AtomicLong maxLatency;
+ // how many retried puts we have inserted in the queue
+ private final AtomicInteger retriedPutsInQueue;
+
+ public HTableFlushWorker(HServerAddress addr, LinkedBlockingQueue<PutStatus> queue) {
this.addr = addr;
- this.conf = conf;
- this.connection = connection;
- this.htableMultiplexer = htableMultiplexer;
this.queue = queue;
this.totalFailedPutCount = new AtomicLong(0);
this.totalSucceededPutCount = new AtomicLong(0);
@@ -592,6 +590,7 @@ public class HTableMultiplexer {
this.avgProcessingPutCount = new AtomicAverageCounter();
this.averageLatency = new AtomicAverageCounter();
this.maxLatency = new AtomicLong(0);
+ this.retriedPutsInQueue = new AtomicInteger(0);
}
public long getTotalFailedCount() {
@@ -628,6 +627,12 @@ public class HTableMultiplexer {
return this.maxLatency.getAndSet(0);
}
+ /**
+ * Resubmit logic for failed put. If we have exhausted the retry count we
+ * return false. Otherwise, we schedule over executor when the failed put is
+ * going to resubmitted. Depending on how many times it has failed, the time
+ * when it is going to be tried again will increase.
+ */
private boolean resubmitFailedPut(PutStatus failedPutStatus, HServerAddress oldLoc) throws IOException{
Put failedPut = failedPutStatus.getPut();
// The currentPut is failed. So get the table name for the currentPut.
@@ -639,9 +644,40 @@ public class HTableMultiplexer {
// Update the failed counter and no retry any more.
return false;
} else {
- // Retry one more time
HBaseRPCOptions options = failedPutStatus.getOptions();
- return this.htableMultiplexer.put(tableName, failedPut, retryCount, options);
+ // schedule the retry of the failed put
+ int currPuts = retriedPutsInQueue.incrementAndGet();
+ // only schedule the put if we are below limit of allowed scheduled-retriable puts
+ if (currPuts <= HTableMultiplexer.this.retriedInQueueMax) {
+ Runnable actuallyReinsert = new Runnable() {
+ @Override
+ public void run() {
+ retriedPutsInQueue.decrementAndGet();
+ try {
+ HTableMultiplexer.this.put(tableName, failedPut, retryCount,
+ options);
+ } catch (IOException e) {
+ // Log all the exceptions and move on
+ LOG.debug("Caught some exceptions " + e
+ + " when reinserting puts to region server "
+ + addr.getHostNameWithPort());
+ }
+ }
+ };
+ // Wait at most DEFAULT_HBASE_RPC_TIMEOUT
+ long waitTimeMs = Math.max(conf.getInt(
+ HConstants.HBASE_RPC_TIMEOUT_KEY,
+ HConstants.DEFAULT_HBASE_RPC_TIMEOUT), (long) (frequency * Math
+ .pow(2, HTableMultiplexer.this.retryNum - retryCount)));
+ executor.schedule(actuallyReinsert,
+ waitTimeMs,
+ TimeUnit.MILLISECONDS);
+ return true;
+ } else {
+ // limit for allowed scheduled-retriable puts is reached, put is failed.
+ retriedPutsInQueue.decrementAndGet(); // decrement since we optimistically incremented
+ return false;
+ }
}
}
@@ -686,43 +722,45 @@ public class HTableMultiplexer {
}
}
+ List<PutStatus> putsForResubmit;
+
// Process this multiput request
- List<Put> failed = null;
Map<String, HRegionFailureInfo> failureInfo = new HashMap<String, HRegionFailureInfo>();
try {
- failed = connection.processListOfMultiPut(Arrays.asList(mput),
+ List<Put> failed = connection.processListOfMultiPut(Arrays.asList(mput),
null, options, failureInfo);
+ if (failed != null) {
+ if (failed.size() == processingList.size()) {
+ // All the puts for this region server are failed. Going to retry
+ // it later
+ putsForResubmit = processingList;
+ } else {
+ putsForResubmit = new ArrayList<>();
+ Set<Put> failedPutSet = new HashSet<Put>(failed);
+ for (PutStatus putStatus : processingList) {
+ if (failedPutSet.contains(putStatus.getPut())) {
+ putsForResubmit.add(putStatus);
+ }
+ }
+ }
+ } else {
+ putsForResubmit = Collections.emptyList();
+ }
} catch (PreemptiveFastFailException e) {
// Client is not blocking on us. So, let us treat this
// as a normal failure, and retry.
- for (PutStatus putStatus : processingList) {
- if (!resubmitFailedPut(putStatus, this.addr)) {
- completelyFailed++;
- }
- }
+ putsForResubmit = processingList;
}
long putsToRetry = 0;
- if (failed != null) {
- if (failed.size() == processingList.size()) {
- // All the puts for this region server are failed. Going to retry
- // it later
- for (PutStatus putStatus : processingList) {
- if (!resubmitFailedPut(putStatus, this.addr)) {
- completelyFailed++;
- }
- }
- } else {
- Set<Put> failedPutSet = new HashSet<Put>(failed);
- for (PutStatus putStatus : processingList) {
- if (failedPutSet.contains(putStatus.getPut())
- && !resubmitFailedPut(putStatus, this.addr)) {
- completelyFailed++;
- }
- }
+ for (PutStatus putStatus : putsForResubmit) {
+ if (!resubmitFailedPut(putStatus, this.addr)) {
+ completelyFailed++;
}
- putsToRetry = failed.size() - completelyFailed;
}
+ putsToRetry = putsForResubmit.size() - completelyFailed;
+
+
// Update the totalFailedCount
this.totalFailedPutCount.addAndGet(completelyFailed);
// Update the totalSucceededPutCount
@@ -756,5 +794,6 @@ public class HTableMultiplexer {
+ addr.getHostNameWithPort());
}
}
+
}
}
[42/50] [abbrv] git commit: [HBASE-9930]
ThriftHRegionServer.setQuorumReadTimeoutMillis was leftover in the
Asyncification process.
Posted by ec...@apache.org.
[HBASE-9930] ThriftHRegionServer.setQuorumReadTimeoutMillis was leftover in the Asyncification process.
Summary: Asyncification ThriftHRegionServer.setQuorumReadTimeoutMillis as well
Test Plan: Run TestHRegionInterfaceSimpleFunctions#setQuorumReadTimeoutMillis.
Reviewers: elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1416404
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43367 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e3de48dd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e3de48dd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e3de48dd
Branch: refs/heads/0.89-fb
Commit: e3de48dd2f76cad41e2934efc6258b0c3450f029
Parents: 78c632a
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jul 8 14:33:18 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../hbase/regionserver/ThriftHRegionServer.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e3de48dd/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
index 1ba2f62..5b2d328 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicLong;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@@ -54,7 +53,6 @@ import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TMultiResponse;
import org.apache.hadoop.hbase.client.TRowMutations;
-import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram;
import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram.Bucket;
import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.ScannerResult;
@@ -69,7 +67,6 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Writable;
-import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
/**
@@ -762,10 +759,14 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
}
@Override
- public ListenableFuture<Void> setHDFSQuorumReadTimeoutMillis(long timeoutMillis) {
- server.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
- return null;
-
+ public ListenableFuture<Void> setHDFSQuorumReadTimeoutMillis(
+ final long timeoutMillis) {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ server.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
+ return null;
+ }
+ });
}
@Override
[36/50] [abbrv] git commit: [master] Remove the dependency of the
RegionServer being alive to clear it from blacklist
Posted by ec...@apache.org.
[master] Remove the dependency of the RegionServer being alive to clear it from blacklist
Summary:
We do not need the regionserver to be alive to clear it from the blacklist.
We just need the 'hostname:port' for this purpose. We already have this in
the serverAddr, so we should just use it
Test Plan: unit tests
Reviewers: manukranthk, aaiyer, adela, pwhite
Reviewed By: pwhite
Subscribers: hbase-dev@
Differential Revision: https://phabricator.fb.com/D1447852
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43632 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/14758a60
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/14758a60
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/14758a60
Branch: refs/heads/0.89-fb
Commit: 14758a60eeda1cfae96e8127fcbf3c5774b1a8b4
Parents: 2cb3ebb
Author: rshroff <rs...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jul 21 22:25:32 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/14758a60/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java b/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
index 84d7209..2537704 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RollingRestart.java
@@ -456,8 +456,7 @@ public class RollingRestart {
private void removeFromBlackList() {
try {
- admin.getMaster().clearBlacklistedServer(
- getHRegionConnection(serverAddr).getHServerInfo().getHostnamePort());
+ admin.getMaster().clearBlacklistedServer(serverAddr.getHostNameWithPort());
} catch (IOException e) {
LOG.error("Failed to remove the server from black list. Please remove it");
}
[11/50] [abbrv] git commit: [HBASE-11254] Fix HTableMultiplexer under
java7
Posted by ec...@apache.org.
[HBASE-11254] Fix HTableMultiplexer under java7
Summary:
Since we work with java8 locally - I didn't notice that this doesn't compile under java7.
This diff is fixing the build. So another point why we shouldn't run with java greater then the java under which we develop.
Test Plan: compileandship
Reviewers: elliott, manukranthk, daviddeng, rshroff, mycnyc
Reviewed By: mycnyc
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1365650
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42683 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ce2595de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ce2595de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ce2595de
Branch: refs/heads/0.89-fb
Commit: ce2595dee0c7e6a9053dc982307829478f1627c7
Parents: d318363
Author: adela <ad...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 5 17:43:41 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/HTableMultiplexer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ce2595de/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
index 9156fe4..a3360c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
@@ -634,17 +634,17 @@ public class HTableMultiplexer {
* when it is going to be tried again will increase.
*/
private boolean resubmitFailedPut(PutStatus failedPutStatus, HServerAddress oldLoc) throws IOException{
- Put failedPut = failedPutStatus.getPut();
+ final Put failedPut = failedPutStatus.getPut();
// The currentPut is failed. So get the table name for the currentPut.
- byte[] tableName = failedPutStatus.getRegionInfo().getTableDesc().getName();
+ final byte[] tableName = failedPutStatus.getRegionInfo().getTableDesc().getName();
// Decrease the retry count
- int retryCount = failedPutStatus.getMaxRetryCount() - 1;
+ final int retryCount = failedPutStatus.getMaxRetryCount() - 1;
if (retryCount <= 0) {
// Update the failed counter and no retry any more.
return false;
} else {
- HBaseRPCOptions options = failedPutStatus.getOptions();
+ final HBaseRPCOptions options = failedPutStatus.getOptions();
// schedule the retry of the failed put
int currPuts = retriedPutsInQueue.incrementAndGet();
// only schedule the put if we are below limit of allowed scheduled-retriable puts
[33/50] [abbrv] git commit: [HBASE-9704] HLog profiling will fail to
work.
Posted by ec...@apache.org.
[HBASE-9704] HLog profiling will fail to work.
Summary: HLog profiling will not work anymore as we move more towards an asynchronous HRegionServer. The profiling is Thread based and assumes that the life of a query belongs to a single thread. We can fix this problem by per request metrics : https://our.intern.facebook.com/intern/wiki/index.php/HBase/InternProjects/HBasePerRequestMetrics.
Test Plan: Ensure TestHeaderSendRecieve works fine.
Reviewers: gauravm, adela
Reviewed By: adela
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1402977
Tasks: 4584691
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43154 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ff5311d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ff5311d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ff5311d
Branch: refs/heads/0.89-fb
Commit: 5ff5311da2ea970552f7e020aae7a5fe6674343e
Parents: abbf77d
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 25 20:32:30 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ff5311d/src/test/java/org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java
index 61e3521..00d1454 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestHeaderSendReceive.java
@@ -161,10 +161,6 @@ public class TestHeaderSendReceive {
Assert.assertTrue(ProfilingData.TOTAL_SERVER_TIME_MS
+ " should be greater than zero",
pd.getLong(ProfilingData.TOTAL_SERVER_TIME_MS).longValue() >= 0L);
- if (isPut) {
- assertNotNull(ProfilingData.HLOG_SYNC_TIME_MS + " is null",
- pd.getLong(ProfilingData.HLOG_SYNC_TIME_MS));
- }
}
/**
[05/50] [abbrv] git commit: [master] add more information in
exception thrown from Store's constructor
Posted by ec...@apache.org.
[master] add more information in exception thrown from Store's constructor
Summary: For debugging the root cause of complains about missing column family directory on searchhbase003
Test Plan: N/A
Reviewers: rshroff, daviddeng, syyang, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1358620
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42601 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d5640c10
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d5640c10
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d5640c10
Branch: refs/heads/0.89-fb
Commit: d5640c10241be58d9bbb7e799f3ecabf10d390eb
Parents: 2d7c39e
Author: fan <fa...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 2 10:31:21 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:43:00 2014 -0700
----------------------------------------------------------------------
src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d5640c10/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 52606df..bc45eb2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -221,10 +221,10 @@ public class Store extends SchemaConfigured implements HeapSize,
if (region != null) {
LOG.info("Creating one");
if (!this.fs.mkdirs(this.homedir))
- throw new IOException("Failed create of: " + this.homedir.toString());
+ throw new IOException("Failed create of: " + this.homedir);
} else {
- throw new IOException("Failed create a read only store. " +
- "Possibly an inconsistent region");
+ throw new IOException("Failed create a read only store on dir "
+ + this.homedir + " . Possibly an inconsistent region." );
}
}
this.family = family;
[43/50] [abbrv] git commit: [HBASE-11486] ThriftHBaseException
modification for RegionOverloadedException
Posted by ec...@apache.org.
[HBASE-11486] ThriftHBaseException modification for RegionOverloadedException
Summary: ThriftHBaseException now contains one more parameter 'waitMilli' to support RegionOverloadedException
Test Plan: Tested by some temp logics, make single put operation directly throws RegionOverloadedException
Reviewers: manukranthk, adela, fan, daviddeng, gauravm
Reviewed By: gauravm
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1421804
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43406 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e1133c6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e1133c6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e1133c6
Branch: refs/heads/0.89-fb
Commit: 8e1133c602cc7c3ee84c950573810da70c3badfb
Parents: e3de48d
Author: xhae <xh...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jul 9 21:07:01 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../thrift/exceptions/ThriftHBaseException.java | 30 ++++++++++++++++----
.../org/apache/hadoop/hbase/thrift/HBase.thrift | 1 +
.../thrift/swift/TestThriftExceptions.java | 20 +++++++------
3 files changed, 37 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e1133c6/src/main/java/org/apache/hadoop/hbase/ipc/thrift/exceptions/ThriftHBaseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/thrift/exceptions/ThriftHBaseException.java b/src/main/java/org/apache/hadoop/hbase/ipc/thrift/exceptions/ThriftHBaseException.java
index a208db7..e877ad9 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/thrift/exceptions/ThriftHBaseException.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/thrift/exceptions/ThriftHBaseException.java
@@ -32,6 +32,7 @@ import com.facebook.swift.codec.ThriftStruct;
import com.google.common.base.Objects;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.RegionException;
import org.apache.hadoop.hbase.util.Bytes;
@ThriftStruct
@@ -44,6 +45,7 @@ public class ThriftHBaseException extends Exception {
private String message;
private String exceptionClass;
private byte[] serializedServerJavaEx;
+ private long waitMillis = 0;
/**
* Swift Contructor used for serialization
@@ -51,14 +53,17 @@ public class ThriftHBaseException extends Exception {
* @param message - the message of the exception
* @param exceptionClass - the class of which instance the exception is
* @param serializedServerJavaEx - serialized java exception
+ * @param waitMillis - required waiting milliseconds for RegionException
*/
@ThriftConstructor
public ThriftHBaseException(@ThriftField(1) String message,
@ThriftField(2) String exceptionClass,
- @ThriftField(3) byte[] serializedServerJavaEx) {
- this.message = message;
- this.exceptionClass = exceptionClass;
- this.serializedServerJavaEx = serializedServerJavaEx;
+ @ThriftField(3) byte[] serializedServerJavaEx,
+ @ThriftField(4) long waitMillis) {
+ this.message = message;
+ this.exceptionClass = exceptionClass;
+ this.serializedServerJavaEx = serializedServerJavaEx;
+ this.waitMillis = waitMillis;
}
public ThriftHBaseException(){}
@@ -71,6 +76,9 @@ public class ThriftHBaseException extends Exception {
this.message = serverJavaException.getMessage();
this.exceptionClass = serverJavaException.getClass().getCanonicalName();
serializedServerJavaEx = out.toByteArray();
+ if (serverJavaException instanceof RegionException) {
+ waitMillis = ((RegionException)serverJavaException).getBackoffTimeMillis();
+ }
} catch (IOException e) {
// Should never happen in reality
LOG.error("Exception happened during serialization of java server exception");
@@ -105,9 +113,14 @@ public class ThriftHBaseException extends Exception {
return serializedServerJavaEx;
}
+ @ThriftField(4)
+ public long getWaitMilli() {
+ return waitMillis;
+ }
+
@Override
public int hashCode() {
- return Objects.hashCode(message, exceptionClass, serializedServerJavaEx);
+ return Objects.hashCode(message, exceptionClass, serializedServerJavaEx, waitMillis);
}
@Override
@@ -138,6 +151,10 @@ public class ThriftHBaseException extends Exception {
return false;
}
+ if (this.getWaitMilli() != otherException.getWaitMilli()) {
+ return false;
+ }
+
return true;
}
@@ -145,6 +162,7 @@ public class ThriftHBaseException extends Exception {
public String toString() {
return getClass() + " [message=" + message + ", exceptionClass="
+ exceptionClass + ", serializedServerJavaEx Hash="
- + Arrays.toString(serializedServerJavaEx).hashCode() + "]";
+ + Arrays.toString(serializedServerJavaEx).hashCode()
+ + ", waitMillis=" + waitMillis + "]";
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e1133c6/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
index 6d19c98..f6a1488 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift/HBase.thrift
@@ -146,6 +146,7 @@ exception ThriftHBaseException {
1: string message;
2: string exceptionClass;
3: binary serializedServerJavaEx;
+ 4: i64 waitMillis;
}
struct MultiAction {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e1133c6/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestThriftExceptions.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestThriftExceptions.java b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestThriftExceptions.java
index ba5282a..ef0779d 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestThriftExceptions.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestThriftExceptions.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionException;
import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.ipc.ThriftHRegionInterface;
@@ -52,16 +53,19 @@ public class TestThriftExceptions {
*/
@Test
public void testSerialization() throws Exception {
- IOException ioe = new IOException("FooBar");
- ThriftHBaseException thriftHBaseException = new ThriftHBaseException(ioe);
- byte[] thriftHBaseExceptionBytes =
- Bytes.writeThriftBytes(thriftHBaseException, ThriftHBaseException.class);
+ IOException[] ioeList = {new IOException("FooBar"),
+ new RegionException("FooBar", 2000)};
+ for(IOException ioe : ioeList) {
+ ThriftHBaseException thriftHBaseException = new ThriftHBaseException(ioe);
+ byte[] thriftHBaseExceptionBytes =
+ Bytes.writeThriftBytes(thriftHBaseException, ThriftHBaseException.class);
- ThriftHBaseException deserThriftHBaseException =
- Bytes.readThriftBytes(thriftHBaseExceptionBytes,
- ThriftHBaseException.class);
+ ThriftHBaseException deserThriftHBaseException =
+ Bytes.readThriftBytes(thriftHBaseExceptionBytes,
+ ThriftHBaseException.class);
- assertEquals(deserThriftHBaseException, thriftHBaseException);
+ assertEquals(deserThriftHBaseException, thriftHBaseException);
+ }
}
[15/50] [abbrv] git commit: [HBASE-11283] Roll HLog if sync time is
anomalous
Posted by ec...@apache.org.
[HBASE-11283] Roll HLog if sync time is anomalous
Summary:
When a WAL gets slow this will schedule a roll.
This diff will also rate limit the forced rolls.
Test Plan: Added testSlowRoll to TestHLog
Reviewers: leon, elenapr, aaiyer, manukranthk
Reviewed By: manukranthk
Subscribers: daviddeng, aleksandr, hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1359586
Tasks: 4422839
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42799 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ae31cc53
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ae31cc53
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ae31cc53
Branch: refs/heads/0.89-fb
Commit: ae31cc53050bdf656bf16f094c0b066eb5a0fc0e
Parents: b848685
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Wed Jun 11 17:18:51 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/LogRoller.java | 22 +++++-
.../hadoop/hbase/regionserver/wal/HLog.java | 20 ++++--
.../hadoop/hbase/regionserver/wal/TestHLog.java | 71 +++++++++++++++++---
3 files changed, 96 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae31cc53/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 357f648..1eb0944 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.hbase.util.HasThread;
@@ -42,14 +43,25 @@ import org.apache.hadoop.hbase.util.HasThread;
*/
public class LogRoller extends HasThread implements LogRollListener {
static final Log LOG = LogFactory.getLog(LogRoller.class);
+
+ /**
+ * The minimum time in between requested log rolls.
+ * In MS.
+ */
+ public static final String LOG_ROLL_REQUEST_PERIOD_KEY =
+ "hbase.regionserver.logroll.request.period";
+ public static final long LOG_ROLL_REQUEST_PERIOD_DEFAULT = TimeUnit.SECONDS.toMillis(60);
+
private final ReentrantLock rollLock = new ReentrantLock();
private final AtomicBoolean rollLog = new AtomicBoolean(false);
private final HRegionServer server;
private volatile long lastrolltime = System.currentTimeMillis();
+ private volatile long lastrollRequestTime = System.currentTimeMillis();
// Period to roll log.
private final long rollperiod;
private final int hlogIndexID;
private final String logRollerName;
+ private final long timeBetweenRequest;
/** @param server */
public LogRoller(final HRegionServer server, int hlogIndexID) {
@@ -57,6 +69,8 @@ public class LogRoller extends HasThread implements LogRollListener {
this.server = server;
this.rollperiod =
this.server.conf.getLong("hbase.regionserver.logroll.period", 3600000);
+ this.timeBetweenRequest = this.server.conf.getLong(LOG_ROLL_REQUEST_PERIOD_KEY,
+ LOG_ROLL_REQUEST_PERIOD_DEFAULT);
this.hlogIndexID = hlogIndexID;
this.logRollerName = "HLogRoller-" + hlogIndexID + " ";
}
@@ -152,9 +166,13 @@ public class LogRoller extends HasThread implements LogRollListener {
}
public void logRollRequested() {
+ long currentTime = System.currentTimeMillis();
synchronized (rollLog) {
- rollLog.set(true);
- rollLog.notifyAll();
+ if ((currentTime - lastrollRequestTime) > timeBetweenRequest) {
+ lastrollRequestTime = currentTime;
+ rollLog.set(true);
+ rollLog.notifyAll();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae31cc53/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index db775d9..044fef4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -158,6 +158,7 @@ public class HLog implements Syncable {
/** We include all timestamps by default */
public static final long DEFAULT_LATEST_TS_TO_INCLUDE = Long.MAX_VALUE;
+ public static final long SECOND_IN_NS = TimeUnit.SECONDS.toNanos(1);
// If enabled, old logs will be archived into hourly sub-directories instead of
// server address sub-directories.
@@ -173,13 +174,15 @@ public class HLog implements Syncable {
private final String prefix;
private final AtomicLong unflushedEntries = new AtomicLong(0);
private final Path oldLogDir;
+ private final int slowBeforeRoll;
+ private final AtomicLong slowSyncs = new AtomicLong(0);
private volatile long syncTillHere = 0;
private final List<LogActionsListener> actionListeners =
Collections.synchronizedList(new ArrayList<LogActionsListener>());
- private static Class<? extends Writer> logWriterClass;
- private static Class<? extends Reader> logReaderClass;
+ static Class<? extends Writer> logWriterClass;
+ static Class<? extends Reader> logReaderClass;
private OutputStream hdfs_out; // OutputStream associated with the current SequenceFile.writer
private int initialReplication; // initial replication factor of SequenceFile.writer
@@ -458,6 +461,8 @@ public class HLog implements Syncable {
this.optionalFlushInterval =
conf.getLong("hbase.regionserver.optionallogflushinterval", 1 * 1000);
+ this.slowBeforeRoll = conf.getInt("hbase.hlog.slow.sync.before.roll", 5);
+
if (!fs.exists(oldLogDir)) {
fs.mkdirs(oldLogDir);
}
@@ -606,6 +611,7 @@ public class HLog implements Syncable {
newWriter.append(new HLog.Entry(key, edit));
syncWriter(newWriter);
}
+ this.slowSyncs.set(0);
} catch (IOException ioe) {
// If we fail to create a new writer, let us clean up the file.
// Do not worry if the delete fails.
@@ -1091,11 +1097,17 @@ public class HLog implements Syncable {
long end = System.nanoTime();
long syncTime = end - start;
gsyncTime.inc(syncTime);
- if (syncTime > 1000000000) {
+ if (syncTime > SECOND_IN_NS) {
LOG.warn(String.format(
- "%s took %d ms appending an edit to hlog; editcount=%d, len~=%s",
+ "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
Thread.currentThread().getName(), syncTime, this.numEntries.get(),
StringUtils.humanReadableInt(len)));
+ slowSyncs.incrementAndGet();
+ }
+
+ if (slowSyncs.get() >= slowBeforeRoll) {
+ this.requestLogRoll();
+ this.slowSyncs.set(0);
}
// Update the per-request profiling data
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae31cc53/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 7916599..b7d644e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -28,6 +28,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.UnstableTests;
import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ExceptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hdfs.DFSClient;
@@ -99,7 +101,10 @@ public class TestHLog {
@After
public void tearDown() throws Exception {
+ HLog.logReaderClass = null;
+ HLog.logWriterClass = null;
}
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Make block sizes small.
@@ -118,19 +123,16 @@ public class TestHLog {
TEST_UTIL.getConfiguration().setInt(
"dfs.client.block.recovery.retries", 1);
TEST_UTIL.getConfiguration().setBoolean(HConstants.HLOG_FORMAT_BACKWARD_COMPATIBILITY, false);
- TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniDFSCluster(3);
conf = TEST_UTIL.getConfiguration();
cluster = TEST_UTIL.getDFSCluster();
fs = cluster.getFileSystem();
- hbaseDir = new Path(TEST_UTIL.getConfiguration().get("hbase.rootdir"));
+ hbaseDir = new Path(fs.getWorkingDirectory(), "hbase");
+ conf.set(HConstants.HBASE_DIR, hbaseDir.toString());
oldLogDir = new Path(hbaseDir, ".oldlogs");
- dir = new Path(hbaseDir, getName());
- }
- private static String getName() {
- // TODO Auto-generated method stub
- return "TestHLog";
+ dir = new Path(hbaseDir, TestHLog.class.getSimpleName());
}
/**
@@ -141,7 +143,7 @@ public class TestHLog {
@Test
public void testSplit() throws IOException {
- final byte [] tableName = Bytes.toBytes(getName());
+ final byte [] tableName = Bytes.toBytes(TestHLog.class.getSimpleName());
final byte [] rowName = tableName;
Path logdir = new Path(dir, HConstants.HREGION_LOGDIR_NAME);
HLog log = new HLog(fs, logdir, oldLogDir, conf, null);
@@ -191,9 +193,9 @@ public class TestHLog {
@Category(UnstableTests.class)
@Test
public void testSync() throws Exception {
- byte [] bytes = Bytes.toBytes(getName());
+ byte [] bytes = Bytes.toBytes(TestHLog.class.getSimpleName());
// First verify that using streams all works.
- Path p = new Path(dir, getName() + ".fsdos");
+ Path p = new Path(dir, TestHLog.class.getSimpleName() + ".fsdos");
FSDataOutputStream out = fs.create(p);
out.write(bytes);
out.sync();
@@ -337,7 +339,7 @@ public class TestHLog {
// 3. HDFS-142 (on restart, maintain pendingCreates)
@Test
public void testAppendClose() throws Exception {
- byte [] tableName = Bytes.toBytes(getName());
+ byte [] tableName = Bytes.toBytes(TestHLog.class.getSimpleName());
HRegionInfo regioninfo = new HRegionInfo(new HTableDescriptor(tableName),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
Path subdir = new Path(dir, "hlogdir");
@@ -503,6 +505,40 @@ public class TestHLog {
}
}
+ @Test
+ public void testSlowRoll() throws Exception {
+ // Make a copy of the conf so that things don't pollute other tests
+ Configuration c = new Configuration(TEST_UTIL.getConfiguration());
+ c.setClass("hbase.regionserver.hlog.writer.impl",
+ SlowSequenceFileLogWriter.class,
+ HLog.Writer.class);
+
+ int syncs = conf.getInt("hbase.hlog.slow.sync.before.roll", 5) * 2;
+
+ final AtomicInteger rollsRequests = new AtomicInteger(0);
+ HLog log = new HLog(fs, dir, oldLogDir, c, new LogRollListener() {
+ @Override public void logRollRequested() {
+ rollsRequests.incrementAndGet();
+ }
+ });
+
+ byte[] table = Bytes.toBytes("table");
+ byte[] cf = Bytes.toBytes("cf");
+ byte[] qual = Bytes.toBytes("qual");
+ byte[] val = Bytes.toBytes("val");
+
+ HRegionInfo info = new HRegionInfo(new HTableDescriptor(table),
+ null,null, false);
+
+ for (int i = 0; i < syncs; i++) {
+ WALEdit kvs = new WALEdit();
+ kvs.add(new KeyValue(Bytes.toBytes(i), cf, qual, HConstants.LATEST_TIMESTAMP, val));
+ log.append(info, table, kvs, System.currentTimeMillis());
+ log.sync(true);
+ }
+ assertEquals("Should request 2 rolls", 2, rollsRequests.get());
+ }
+
/**
* @throws IOException
*/
@@ -615,6 +651,19 @@ public class TestHLog {
}
}
+
+ public static class SlowSequenceFileLogWriter extends SequenceFileLogWriter implements HLog.Writer {
+ @Override
+ public void sync() throws IOException {
+ try {
+ Thread.sleep(1100);
+ } catch (InterruptedException e) {
+ ExceptionUtils.toIOException(e);
+ }
+ super.sync();
+ }
+ }
+
static class DumbLogEntriesVisitor implements LogEntryVisitor {
int increments = 0;
[22/50] [abbrv] git commit: [master] Discard HBaseAdmin during
shutting down the cluster.
Posted by ec...@apache.org.
[master] Discard HBaseAdmin during shutting down the cluster.
Summary: `HBaseAdmin` instance is cached in `HBaseTestingUtility.hbaseAdmin`. During shutting down some of the connection it holds is closed which causes mistake if it is later used again. (`java.io.IOException: The client is stopped`)
Test Plan: `TestHBaseTestingUtility`
Reviewers: manukranthk, adela, gauravm, elliott
Reviewed By: elliott
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1392480
Tasks: 3269413
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42982 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0dbf011
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0dbf011
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0dbf011
Branch: refs/heads/0.89-fb
Commit: b0dbf011bc74e1909e348bfa5c0aacba5cc39f5e
Parents: af24cea
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 19 17:54:09 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/HBaseTestingUtility.java | 3 ++
.../hadoop/hbase/TestHBaseTestingUtility.java | 42 ++++++++++++++++++++
2 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0dbf011/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 10762fb..649512f 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -537,6 +537,9 @@ public class HBaseTestingUtility {
// Wait till hbase is down before going on to shutdown zk.
this.hbaseCluster.join();
this.hbaseCluster = null;
+ // hbaseAdmin holds some connection that have been closed during shutting
+ // down HBase cluster.
+ this.hbaseAdmin = null;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0dbf011/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
new file mode 100644
index 0000000..5a16b11
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2014 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.junit.Test;
+
+/**
+ * Testcase for HBaseTestingUtility.
+ */
+public class TestHBaseTestingUtility {
+ /**
+ * Make sure HBaseAdmin is not reused after the mini-cluster is shut down.
+ */
+ @Test
+ public void testHBaseAdminNoReuse() throws Exception {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ util.startMiniCluster();
+ util.getHBaseAdmin().createTable(new HTableDescriptor("test"));
+ util.shutdownMiniCluster();
+
+ util.startMiniCluster();
+ util.getHBaseAdmin().createTable(new HTableDescriptor("test"));
+ util.shutdownMiniCluster();
+ }
+}
[32/50] [abbrv] git commit: [HBASE-11416] Add size aware
ThriftHRegionServer
Posted by ec...@apache.org.
[HBASE-11416] Add size aware ThriftHRegionServer
Summary:
- Add Async ThriftHRegionServer.
- Added read and write call queues so that writes will be seperated from reads.
- Added size aware ThriftHRegionServer that will throw errors if the call queue gets too large.
Test Plan: Test on Dev cluster soon.
Reviewers: rshroff, manukranthk
Reviewed By: rshroff
Subscribers: abbyedwards, hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1404826
Tasks: 4589584
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43186 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6b30de6e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6b30de6e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6b30de6e
Branch: refs/heads/0.89-fb
Commit: 6b30de6e381c805315f5bc9a61550bca41ae6ade
Parents: 5ff5311
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 26 22:42:11 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 2 +-
.../SizeAwareThriftHRegionServer.java | 148 +++
.../hbase/regionserver/ThriftHRegionServer.java | 993 ++++++++++++-------
.../apache/hadoop/hbase/MiniHBaseCluster.java | 5 +-
.../FailureInjectingThriftHRegionServer.java | 12 +-
.../TestHRegionInterfaceSimpleFunctions.java | 34 +-
7 files changed, 798 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index ddaabce..3970bc2 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -833,7 +833,7 @@ public final class HConstants {
* Number of threads for swift server
*/
public static final String SWIFT_WORKER_THREADS = "hbase.swift.worker.threads";
- public static final int SWIFT_WORKER_THREADS_DEFAULT = 300;
+ public static final int SWIFT_WORKER_THREADS_DEFAULT = 10;
/**
* Number of io threads for swift server
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 3f3947d..7c1c736 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -638,7 +638,7 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
HConstants.DEFAULT_REGIONSERVER_SWIFT_PORT);
Class<? extends ThriftHRegionServer> thriftServerClass =
(Class<? extends ThriftHRegionServer>)
- conf.getClass(HConstants.THRIFT_REGION_SERVER_IMPL, ThriftHRegionServer.class);
+ conf.getClass(HConstants.THRIFT_REGION_SERVER_IMPL, SizeAwareThriftHRegionServer.class);
ThriftHRegionServer thriftServer;
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
new file mode 100644
index 0000000..d2526c6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SizeAwareThriftHRegionServer.java
@@ -0,0 +1,148 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.MultiAction;
+import org.apache.hadoop.hbase.client.MultiPut;
+import org.apache.hadoop.hbase.client.MultiPutResponse;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.TMultiResponse;
+
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SizeAwareThriftHRegionServer extends ThriftHRegionServer {
+
+ private final AtomicLong writeQueueSizebytes = new AtomicLong(0);
+ private final long maxWriteSize;
+
+ public SizeAwareThriftHRegionServer(HRegionServer server) {
+ super(server);
+ this.maxWriteSize = server.getConfiguration().getLong(
+ HConstants.MAX_CALL_QUEUE_MEMORY_SIZE_STRING, HConstants.MAX_CALL_QUEUE_MEMORY_SIZE);
+ }
+
+ @Override
+ public ListenableFuture<Void> put(final byte[] regionName, final Put put) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ final long heapSize = put.heapSize();
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<Void> f = super.put(regionName, put);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ @Override
+ public ListenableFuture<Integer> putRows(final byte[] regionName, final List<Put> puts) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ long heapSize = 0;
+ for (Put p : puts) {
+ heapSize += p.heapSize();
+ }
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<Integer> f = super.putRows(regionName, puts);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ @Override
+ public ListenableFuture<TMultiResponse> multiAction(final MultiAction multi) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ long heapSize = 0;
+ if (multi.getPuts() != null) {
+ for (List<Put> puts : multi.getPuts().values()) {
+ for (Put p : puts) {
+ heapSize += p.heapSize();
+ }
+ }
+ }
+ if (multi.getDeletes() != null) {
+ for (List<Delete> deletes:multi.getDeletes().values()) {
+ for (Delete d:deletes) {
+ heapSize += d.getRow().length * d.size();
+ }
+ }
+ }
+
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<TMultiResponse> f = super.multiAction(multi);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ @Override
+ public ListenableFuture<MultiPutResponse> multiPut(final MultiPut puts) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ long heapSize = 0;
+ for (List<Put> putList : puts.getPuts().values()) {
+ for (Put p : putList) {
+ heapSize += p.heapSize();
+ }
+ }
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<MultiPutResponse> f = super.multiPut(puts);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ @Override
+ public ListenableFuture<Void> processDelete(final byte[] regionName, final Delete delete) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ long heapSize = delete.size() * delete.getRow().length;
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<Void> f = super.processDelete(regionName, delete);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ @Override
+ public ListenableFuture<Integer> processListOfDeletes(final byte[] regionName, final List<Delete> deletes) {
+ if (writeQueueSizebytes.get() < maxWriteSize) {
+ long heapSize = 0;
+ for (Delete d:deletes) {
+ heapSize += d.size() * d.getRow().length;
+ }
+ writeQueueSizebytes.addAndGet(heapSize);
+ ListenableFuture<Integer> f = super.processListOfDeletes(regionName, deletes);
+ Futures.addCallback(f, new DecrementWriteSizeCallback(heapSize));
+ return f;
+ } else {
+ return Futures.immediateFailedFuture(new RegionOverloadedException());
+ }
+ }
+
+ private class DecrementWriteSizeCallback<V> implements FutureCallback<V> {
+ private final long heapSize;
+
+ public DecrementWriteSizeCallback(long heapSize) {
+ this.heapSize = heapSize;
+ }
+
+ @Override public void onSuccess(@Nullable V aVoid) {
+ writeQueueSizebytes.addAndGet(-1 * heapSize);
+ }
+
+ @Override public void onFailure(Throwable throwable) {
+ writeQueueSizebytes.addAndGet(-1 * heapSize);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
index eeea3df..0d6eaca 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
@@ -27,9 +27,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -48,12 +54,14 @@ import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TMultiResponse;
import org.apache.hadoop.hbase.client.TRowMutations;
+import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram;
import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram.Bucket;
import org.apache.hadoop.hbase.ipc.ScannerResult;
import org.apache.hadoop.hbase.ipc.ThriftHRegionInterface;
import org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException;
import org.apache.hadoop.hbase.master.AssignmentPlan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.DaemonThreadFactory;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
@@ -67,477 +75,639 @@ import com.google.common.util.concurrent.ListenableFuture;
* This is just a wrapper around {@link HRegionServer}
*
*/
-public class ThriftHRegionServer implements ThriftHRegionInterface.Sync {
+public class ThriftHRegionServer implements ThriftHRegionInterface.Async {
private static Log LOG = LogFactory.getLog(ThriftHRegionServer.class);
private HRegionServer server;
public ThriftHRegionServer(HRegionServer server) {
this.server = server;
- }
+ final Configuration configuration = server.getConfiguration();
- @Override
- public HRegionInfo getRegionInfo(byte[] regionName)
- throws ThriftHBaseException {
- try {
- HRegionInfo r = server.getRegionInfo(regionName);
- LOG.debug("Printing the result of getClosestRowOrBefore : " + r);
- return r;
- } catch (NotServingRegionException e) {
- e.printStackTrace();
- throw new ThriftHBaseException(e);
- }
+ // Current default assumes 80% writes to 20% reads.
+ int readHandlers = configuration.getInt("hbase.regionserver.handler.read.count",
+ configuration.getInt("hbase.regionserver.handler.count", 300) / 5 );
+ int writeHandlers = configuration.getInt("hbase.regionserver.handler.write.count",
+ configuration.getInt("hbase.regionserver.handler.count", 300) - readHandlers );
+
+ writeService = MoreExecutors.listeningDecorator(
+ Executors.newFixedThreadPool(writeHandlers,
+ new DaemonThreadFactory("HBase-Write-Handler-")));
+
+ readService = MoreExecutors.listeningDecorator(
+ Executors.newFixedThreadPool(readHandlers,
+ new DaemonThreadFactory("HBase-Read-Handler-")));
}
+ private final ListeningExecutorService writeService;
+
+
+ private final ListeningExecutorService readService;
+
+
@Override
- public Result getClosestRowBefore(byte[] regionName, byte[] row, byte[] family)
- throws ThriftHBaseException {
- try {
- Result r = server.getClosestRowBefore(regionName, row, family);
- if (r == null) {
- return Result.SENTINEL_RESULT;
- } else {
- return addThriftRegionInfoQualifierIfNeeded(r);
+ public ListenableFuture<HRegionInfo> getRegionInfo(final byte[] regionName) {
+ return readService.submit(new Callable<HRegionInfo>() {
+ @Override public HRegionInfo call() throws Exception {
+ try {
+ HRegionInfo r = server.getRegionInfo(regionName);
+ LOG.debug("Printing the result of getClosestRowOrBefore : " + r);
+ return r;
+ } catch (NotServingRegionException e) {
+ throw new ThriftHBaseException(e);
+ }
}
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ });
}
- /**
- * If there is a 'regioninfo' value in the result, it would be in Writable
- * serialized form. Add a thrift serialized HRegionInfo object for the
- * non-Java objects.
- */
- private Result addThriftRegionInfoQualifierIfNeeded(Result tentativeResult) {
- //TODO: Thrift has some problem serializing HRegionInfo. Since this method is only
- // for C++ client, I temporarily disable it. After the problem is fixed we should
- // remove the flag.
- if (HConstants.DISABLE_THRIFT_REGION_INFO_QUALIFIER) {
- return tentativeResult;
- }
+ @Override
+ public ListenableFuture<Result> getClosestRowBefore(final byte[] regionName, final byte[] row,
+ final byte[] family) {
+ return readService.submit(new Callable<Result>(){
- // Get the serialized HRegionInfo object
- byte[] value = tentativeResult.searchValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
- // If the value exists, then we need to do something, otherwise, we return
- // the result untouched.
- if (value != null && value.length > 0) {
- try {
- // Get the Writable-serialized HRegionInfo object.
- HRegionInfo hri = (HRegionInfo) Writables.getWritable(value,
- new HRegionInfo());
- byte[] thriftSerializedHri =
- Bytes.writeThriftBytes(hri, HRegionInfo.class);
- // Create the KV with the thrift-serialized HRegionInfo
- List<KeyValue> kvList = tentativeResult.getKvs();
- KeyValue kv = new KeyValue(kvList.get(0).getRow(),
- HConstants.CATALOG_FAMILY, HConstants.THRIFT_REGIONINFO_QUALIFIER,
- thriftSerializedHri);
- kvList.add(kv);
- return new Result(kvList);
- } catch (Exception e) {
- // If failed, log the mistake and returns the original result
- LOG.error("Thrift Serialization of the HRegionInfo object failed!");
- e.printStackTrace();
+ @Override public Result call() throws Exception {
+ try {
+ Result r = server.getClosestRowBefore(regionName, row, family);
+ if (r == null) {
+ return Result.SENTINEL_RESULT;
+ } else {
+ return addThriftRegionInfoQualifierIfNeeded(r);
+ }
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
}
- }
- return tentativeResult;
+ });
+
}
@Override
- public void flushRegion(byte[] regionName)
- throws ThriftHBaseException {
- try {
- server.flushRegion(regionName);
- } catch (Exception e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> flushRegion(final byte[] regionName) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.flushRegion(regionName);
+ } catch (Exception e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public void flushRegion(byte[] regionName, long ifOlderThanTS)
- throws ThriftHBaseException {
- try {
- server.flushRegion(regionName, ifOlderThanTS);
- } catch (IllegalArgumentException e) {
- throw new ThriftHBaseException(e);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> flushRegion(final byte[] regionName, final long ifOlderThanTS) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.flushRegion(regionName, ifOlderThanTS);
+ } catch (IllegalArgumentException e) {
+ throw new ThriftHBaseException(e);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public long getLastFlushTime(byte[] regionName) {
- return server.getLastFlushTime(regionName);
+ public ListenableFuture<Long> getLastFlushTime(final byte[] regionName) {
+ return readService.submit(new Callable<Long>() {
+ @Override public Long call() throws Exception {
+ return server.getLastFlushTime(regionName);
+ }
+ });
}
@Override
- public Map<byte[], Long> getLastFlushTimes() {
- MapWritable mapWritable = server.getLastFlushTimes();
- Map<byte[], Long> map = new HashMap<byte[], Long>();
+ public ListenableFuture<Map<byte[], Long>> getLastFlushTimes() {
+ return readService.submit(new Callable<Map<byte[], Long>>() {
+ @Override public Map<byte[], Long> call() throws Exception {
+ MapWritable mapWritable = server.getLastFlushTimes();
+ Map<byte[], Long> map = new HashMap<>();
- for (Entry<Writable, Writable> e : mapWritable.entrySet()) {
- map.put(((BytesWritable) e.getKey()).getBytes(),
- ((LongWritable) e.getValue()).get());
- }
- return map;
+ for (Entry<Writable, Writable> e : mapWritable.entrySet()) {
+ map.put(((BytesWritable) e.getKey()).getBytes(),
+ ((LongWritable) e.getValue()).get());
+ }
+ return map;
+ }
+ });
}
@Override
- public long getCurrentTimeMillis() {
- return server.getCurrentTimeMillis();
+ public ListenableFuture<Long> getCurrentTimeMillis() {
+ return readService.submit(new Callable<Long>() {
+ @Override public Long call() throws Exception {
+ return server.getCurrentTimeMillis();
+ }
+ });
}
@Override
- public long getStartCode() {
- return server.getStartCode();
+ public ListenableFuture<Long> getStartCode() {
+ return readService.submit(new Callable<Long>() {
+ @Override public Long call() throws Exception {
+ return server.getStartCode();
+ }
+ });
}
@Override
- public List<String> getStoreFileList(byte[] regionName, byte[] columnFamily)
- throws ThriftHBaseException {
- try {
- return server.getStoreFileList(regionName, columnFamily);
- } catch (IllegalArgumentException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<String>> getStoreFileList(final byte[] regionName, final byte[] columnFamily) {
+ return readService.submit(new Callable<List<String>>() {
+ @Override public List<String> call() throws Exception {
+ try {
+ return server.getStoreFileList(regionName, columnFamily);
+ } catch (IllegalArgumentException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<String> getStoreFileListForColumnFamilies(byte[] regionName,
- List<byte[]> columnFamilies) throws ThriftHBaseException {
- try {
- byte[][] columnFamiliesArray = new byte[columnFamilies.size()][];
- return server.getStoreFileList(regionName, columnFamilies.toArray(columnFamiliesArray));
- } catch (IllegalArgumentException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<String>> getStoreFileListForColumnFamilies(final byte[] regionName,
+ final List<byte[]> columnFamilies) {
+ return readService.submit(new Callable<List<String>>() {
+ @Override public List<String> call() throws Exception {
+ try {
+ byte[][] columnFamiliesArray = new byte[columnFamilies.size()][];
+ return server.getStoreFileList(regionName, columnFamilies.toArray(columnFamiliesArray));
+ } catch (IllegalArgumentException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<String> getStoreFileListForAllColumnFamilies(byte[] regionName)
- throws ThriftHBaseException {
- try {
- return server.getStoreFileList(regionName);
- } catch (IllegalArgumentException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<String>> getStoreFileListForAllColumnFamilies(final byte[] regionName) {
+ return readService.submit(new Callable<List<String>>() {
+ @Override public List<String> call() throws Exception {
+ try {
+ return server.getStoreFileList(regionName);
+ } catch (IllegalArgumentException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<String> getHLogsList(boolean rollCurrentHLog)
- throws ThriftHBaseException {
- try {
- return server.getHLogsList(rollCurrentHLog);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<String>> getHLogsList(final boolean rollCurrentHLog) {
+ return readService.submit(new Callable<List<String>>() {
+ @Override public List<String> call() throws Exception {
+ try {
+ return server.getHLogsList(rollCurrentHLog);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public Result get(byte[] regionName, Get get) throws ThriftHBaseException {
- try {
- return server.get(regionName, get);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Result> get(final byte[] regionName, final Get get) {
+ return readService.submit(new Callable<Result>() {
+ @Override public Result call() throws Exception {
+ try {
+ return server.get(regionName, get);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<Result> getRows(byte[] regionName, List<Get> gets)
- throws ThriftHBaseException {
- try {
- List<Result> resultList = new ArrayList<>();
- Collections.addAll(resultList, server.get(regionName, gets));
- return resultList;
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<Result>> getRows(final byte[] regionName, final List<Get> gets) {
+ return readService.submit(new Callable<List<Result>>() {
+ @Override public List<Result> call() throws Exception {
+ try {
+ List<Result> resultList = new ArrayList<>();
+ Collections.addAll(resultList, server.get(regionName, gets));
+ return resultList;
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public boolean exists(byte[] regionName, Get get)
- throws ThriftHBaseException {
- try {
- return server.exists(regionName, get);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Boolean> exists(final byte[] regionName, final Get get){
+ return readService.submit(new Callable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ try {
+ return server.exists(regionName, get);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public void put(byte[] regionName, Put put) throws ThriftHBaseException {
- try {
- server.put(regionName, put);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
- }
+ public ListenableFuture<Void> put(final byte[] regionName, final Put put) {
- @Override
- public int putRows(byte[] regionName, List<Put> puts)
- throws ThriftHBaseException {
- try {
- return server.put(regionName, puts);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.put(regionName, put);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public void processDelete(byte[] regionName, Delete delete)
- throws ThriftHBaseException {
- try {
- server.delete(regionName, delete);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Integer> putRows(final byte[] regionName, final List<Put> puts) {
+ return writeService.submit(new Callable<Integer>() {
+ @Override public Integer call() throws Exception {
+ try {
+ return server.put(regionName, puts);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public int processListOfDeletes(byte[] regionName, List<Delete> deletes)
- throws ThriftHBaseException {
- try {
- return server.delete(regionName, deletes);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> processDelete(final byte[] regionName, final Delete delete) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.delete(regionName, delete);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public boolean checkAndPut(byte[] regionName, byte[] row, byte[] family,
- byte[] qualifier, byte[] value, Put put) throws ThriftHBaseException {
- try {
- return server.checkAndPut(regionName, row, family, qualifier, value, put);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Integer> processListOfDeletes(final byte[] regionName, final List<Delete> deletes) {
+ return writeService.submit(new Callable<Integer>() {
+ @Override public Integer call() throws Exception {
+ try {
+ return server.delete(regionName, deletes);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public boolean checkAndDelete(byte[] regionName, byte[] row, byte[] family,
- byte[] qualifier, byte[] value, Delete delete)
- throws ThriftHBaseException {
- try {
- return server.checkAndDelete(regionName, row, family, qualifier, value,
- delete);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Boolean> checkAndPut(final byte[] regionName, final byte[] row, final byte[] family,
+ final byte[] qualifier, final byte[] value, final Put put) {
+
+ return writeService.submit(new Callable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ try {
+ return server.checkAndPut(regionName, row, family, qualifier, value, put);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public long incrementColumnValue(byte[] regionName, byte[] row,
- byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
- throws ThriftHBaseException {
- try {
- return server.incrementColumnValue(regionName, row, family, qualifier, amount, writeToWAL);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Boolean> checkAndDelete(final byte[] regionName, final byte[] row,
+ final byte[] family, final byte[] qualifier, final byte[] value, final Delete delete) {
+
+ return writeService.submit(new Callable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ try {
+ return server.checkAndDelete(regionName, row, family, qualifier, value,
+ delete);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public long openScanner(byte[] regionName, Scan scan)
- throws ThriftHBaseException {
- try {
- return server.openScanner(regionName, scan);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Long> incrementColumnValue(final byte[] regionName, final byte[] row,
+ final byte[] family, final byte[] qualifier, final long amount, final boolean writeToWAL) {
+
+ return writeService.submit(new Callable<Long>() {
+ @Override public Long call() throws Exception {
+ try {
+ return server.incrementColumnValue(regionName, row, family, qualifier, amount, writeToWAL);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public void mutateRow(byte[] regionName, TRowMutations arm)
- throws ThriftHBaseException {
- try {
- server.mutateRow(regionName, RowMutations.Builder.createFromTRowMutations(arm));
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Long> openScanner(final byte[] regionName, final Scan scan) {
+ return readService.submit(new Callable<Long>() {
+ @Override public Long call() throws Exception {
+ try {
+ return server.openScanner(regionName, scan);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public void mutateRows(byte[] regionName, List<TRowMutations> armList)
- throws ThriftHBaseException {
- try {
- List<RowMutations> rowMutations = new ArrayList<>();
- for (TRowMutations mutation : armList) {
- rowMutations.add(RowMutations.Builder.createFromTRowMutations(mutation));
+ public ListenableFuture<Void> mutateRow(final byte[] regionName, final TRowMutations arm) {
+
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.mutateRow(regionName, RowMutations.Builder.createFromTRowMutations(arm));
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
}
- server.mutateRow(regionName, rowMutations);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Void> mutateRows(final byte[] regionName, final List<TRowMutations> armList) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ List<RowMutations> rowMutations = new ArrayList<>();
+ for (TRowMutations mutation : armList) {
+ rowMutations.add(RowMutations.Builder.createFromTRowMutations(mutation));
+ }
+ server.mutateRow(regionName, rowMutations);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
@Deprecated
- public Result next(long scannerId) throws ThriftHBaseException {
- try {
- return server.next(scannerId);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Result> next(final long scannerId) {
+ return readService.submit(new Callable<Result>() {
+ @Override public Result call() throws Exception {
+ try {
+ return server.next(scannerId);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<List<Result>> nextRows(final long scannerId, final int numberOfRows) {
+ return readService.submit(new Callable<List<Result>>() {
+ @Override public List<Result> call() throws Exception {
+ try {
+ Result[] result = server.nextInternal(scannerId, numberOfRows);
+ List<Result> resultList = new ArrayList<>(result.length);
+ for (int i = 0; i < result.length; i ++) {
+ resultList.add(addThriftRegionInfoQualifierIfNeeded(result[i]));
+ }
+ return resultList;
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<Result> nextRows(long scannerId, int numberOfRows)
- throws ThriftHBaseException {
- try {
- Result[] result = server.nextInternal(scannerId, numberOfRows);
- List<Result> resultList = new ArrayList<>(result.length);
- for (int i = 0; i < result.length; i ++) {
- resultList.add(this.addThriftRegionInfoQualifierIfNeeded(result[i]));
+ public ListenableFuture<Void> close(final long scannerId) {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.close(scannerId);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
}
- return resultList;
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ });
}
@Override
- public void close(long scannerId) throws ThriftHBaseException {
- try {
- server.close(scannerId);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<RowLock> lockRow(final byte[] regionName, final byte[] row) {
+ return readService.submit(new Callable<RowLock>() {
+ @Override public RowLock call() throws Exception {
+ try {
+ return new RowLock(row, server.lockRow(regionName, row));
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public RowLock lockRow(byte[] regionName, byte[] row)
- throws ThriftHBaseException {
- try {
- return new RowLock(row, server.lockRow(regionName, row));
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> unlockRow(final byte[] regionName, final long lockId) {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.unlockRow(regionName, lockId);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public void unlockRow(byte[] regionName, long lockId)
- throws ThriftHBaseException {
- try {
- server.unlockRow(regionName, lockId);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
- }
+ public ListenableFuture<List<HRegionInfo>> getRegionsAssignment() {
- @Override
- public List<HRegionInfo> getRegionsAssignment() throws ThriftHBaseException {
- try {
- return Arrays.asList(server.getRegionsAssignment());
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ return readService.submit(new Callable<List<HRegionInfo>>() {
+ @Override public List<HRegionInfo> call() throws Exception {
+ try {
+ return Arrays.asList(server.getRegionsAssignment());
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public HServerInfo getHServerInfo() throws ThriftHBaseException {
- try {
- return server.getHServerInfo();
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<HServerInfo> getHServerInfo() {
+ return readService.submit(new Callable<HServerInfo>() {
+ @Override public HServerInfo call() throws Exception {
+ try {
+ return server.getHServerInfo();
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public TMultiResponse multiAction(MultiAction multi)
- throws ThriftHBaseException {
- try {
- return TMultiResponse.Builder.createFromMultiResponse(server
- .multiAction(multi));
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<TMultiResponse> multiAction(final MultiAction multi) {
+ return writeService.submit(new Callable<TMultiResponse>() {
+ @Override public TMultiResponse call() throws Exception {
+ try {
+ return TMultiResponse.Builder.createFromMultiResponse(server
+ .multiAction(multi));
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public MultiPutResponse multiPut(MultiPut puts) throws ThriftHBaseException {
- try {
- return server.multiPut(puts);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<MultiPutResponse> multiPut(final MultiPut puts) {
+
+ return writeService.submit(new Callable<MultiPutResponse>() {
+ @Override public MultiPutResponse call() throws Exception {
+ try {
+ return server.multiPut(puts);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public void bulkLoadHFile(String hfilePath, byte[] regionName,
- byte[] familyName) throws ThriftHBaseException {
- try {
- server.bulkLoadHFile(hfilePath, regionName, familyName);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> bulkLoadHFile(final String hfilePath, final byte[] regionName,
+ final byte[] familyName) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.bulkLoadHFile(hfilePath, regionName, familyName);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public void bulkLoadHFile(String hfilePath, byte[] regionName,
- byte[] familyName, boolean assignSeqNum) throws ThriftHBaseException {
- try {
- server.bulkLoadHFile(hfilePath, regionName, familyName, assignSeqNum);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> bulkLoadHFile(final String hfilePath, final byte[] regionName,
+ final byte[] familyName, final boolean assignSeqNum) {
+ return writeService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.bulkLoadHFile(hfilePath, regionName, familyName, assignSeqNum);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public void closeRegion(HRegionInfo hri, boolean reportWhenCompleted)
- throws ThriftHBaseException {
- try {
- server.closeRegion(hri, reportWhenCompleted);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Void> closeRegion(final HRegionInfo hri, final boolean reportWhenCompleted) {
+
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ server.closeRegion(hri, reportWhenCompleted);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ return null;
+ }
+ });
}
@Override
- public int updateFavoredNodes(AssignmentPlan plan) throws ThriftHBaseException {
- try {
- return server.updateFavoredNodes(plan);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<Integer> updateFavoredNodes(final AssignmentPlan plan) {
+ return readService.submit(new Callable<Integer>() {
+ @Override public Integer call() throws Exception {
+ try {
+ return server.updateFavoredNodes(plan);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public void updateConfiguration() {
- server.updateConfiguration();
+ public ListenableFuture<Void> updateConfiguration() {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ server.updateConfiguration();
+ return null;
+ }
+ });
}
@Override
- public void stop(String why) {
- server.stop(why);
+ public ListenableFuture<Void> stop(final String why) {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ server.stop(why);
+ return null;
+ }
+ });
}
@Override
- public String getStopReason() {
- return server.getStopReason();
+ public ListenableFuture<String> getStopReason() {
+ return readService.submit(new Callable<String>() {
+ @Override public String call() throws Exception {
+ return server.getStopReason();
+ }
+ });
}
@Override
- public void setNumHDFSQuorumReadThreads(int maxThreads) {
- server.setNumHDFSQuorumReadThreads(maxThreads);
+ public ListenableFuture<Void> setNumHDFSQuorumReadThreads(final int maxThreads) {
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ server.setNumHDFSQuorumReadThreads(maxThreads);
+ return null;
+ }
+ });
}
@Override
- public void setHDFSQuorumReadTimeoutMillis(long timeoutMillis) {
+ public ListenableFuture<Void> setHDFSQuorumReadTimeoutMillis(long timeoutMillis) {
server.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
+ return null;
+
}
@Override
- public boolean isStopped() {
- return server.isStopped();
+ public ListenableFuture<Boolean> isStopped() {
+ return readService.submit(new Callable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ return server.isStopped();
+ }
+ });
}
@Override
- public void stopForRestart() {
- server.stopForRestart();
+ public ListenableFuture<Void> stopForRestart() {
+
+ return readService.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ server.stopForRestart();
+ return null;
+ }
+ });
}
@Override
@@ -546,76 +716,157 @@ public class ThriftHRegionServer implements ThriftHRegionInterface.Sync {
}
@Override
- public String getConfProperty(String paramName) throws ThriftHBaseException {
- return server.getConfProperty(paramName);
+ public ListenableFuture<String> getConfProperty(final String paramName) {
+ return readService.submit(new Callable<String>() {
+ @Override public String call() throws Exception {
+ return server.getConfProperty(paramName);
+ }
+ });
}
@Override
- public List<Bucket> getHistogram(byte[] regionName)
- throws ThriftHBaseException {
- try {
- List<Bucket> buckets = server.getHistogram(regionName);
- if (buckets == null) return new ArrayList<Bucket>();
- return buckets;
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<Bucket>> getHistogram(final byte[] regionName) {
+ return readService.submit(new Callable<List<Bucket>>() {
+ @Override public List<Bucket> call() throws Exception {
+ try {
+ List<Bucket> buckets = server.getHistogram(regionName);
+ if (buckets == null) return new ArrayList<Bucket>();
+ return buckets;
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public List<Bucket> getHistogramForStore(byte[] regionName, byte[] family)
- throws ThriftHBaseException {
- try {
- List<Bucket> buckets = server.getHistogramForStore(regionName, family);
- if (buckets == null) return new ArrayList<Bucket>();
- return buckets;
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<Bucket>> getHistogramForStore(final byte[] regionName, final byte[] family) {
+ return readService.submit(new Callable<List<Bucket>>() {
+ @Override public List<Bucket> call() throws Exception {
+ try {
+ List<Bucket> buckets = server.getHistogramForStore(regionName, family);
+ if (buckets == null) return new ArrayList<Bucket>();
+ return buckets;
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public byte[] callEndpoint(String epName, String methodName,
- List<byte[]> params, final byte[] regionName, final byte[] startRow,
- final byte[] stopRow) throws ThriftHBaseException, IOException {
- return server.callEndpoint(epName, methodName, params, regionName,
- startRow, stopRow);
+ public ListenableFuture<byte[]> callEndpoint(final String epName, final String methodName,
+ final List<byte[]> params, final byte[] regionName, final byte[] startRow,
+ final byte[] stopRow) {
+
+ return readService.submit(new Callable<byte[]>() {
+ @Override public byte[] call() throws Exception {
+ try {
+ return server.callEndpoint(epName, methodName, params, regionName,
+ startRow, stopRow);
+ } catch (IOException ioe) {
+ throw new ThriftHBaseException(ioe);
+ }
+ }
+ });
}
@Override
- public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
- throws ThriftHBaseException {
- try {
- return server.getHistograms(regionNames);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<List<List<Bucket>>> getHistograms(final List<byte[]> regionNames) {
+ return readService.submit(new Callable<List<List<Bucket>>>() {
+ @Override public List<List<Bucket>> call() throws Exception {
+ try {
+ return server.getHistograms(regionNames);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public HRegionLocation getLocation(byte[] table, byte[] row, boolean reload)
- throws ThriftHBaseException {
- try {
- return server.getLocation(table, row, reload);
- } catch (IOException e) {
- throw new ThriftHBaseException(e);
- }
+ public ListenableFuture<HRegionLocation> getLocation(
+ final byte[] table, final byte[] row, final boolean reload) {
+
+ return readService.submit(new Callable<HRegionLocation>() {
+ @Override public HRegionLocation call() throws Exception {
+ try {
+ return server.getLocation(table, row, reload);
+ } catch (IOException e) {
+ throw new ThriftHBaseException(e);
+ }
+ }
+ });
}
@Override
- public ScannerResult scanOpen(byte[] regionName, Scan scan, int numberOfRows)
- throws ThriftHBaseException {
- return server.scanOpen(regionName, scan, numberOfRows);
+ public ListenableFuture<ScannerResult> scanOpen(
+ final byte[] regionName, final Scan scan, final int numberOfRows) {
+
+ return readService.submit(new Callable<ScannerResult>() {
+ @Override public ScannerResult call() throws Exception {
+ return server.scanOpen(regionName, scan, numberOfRows);
+ }
+ });
}
@Override
- public ScannerResult scanNext(long id, int numberOfRows)
- throws ThriftHBaseException {
- return server.scanNext(id, numberOfRows);
+ public ListenableFuture<ScannerResult> scanNext(final long id, final int numberOfRows) {
+
+ return readService.submit(new Callable<ScannerResult>() {
+ @Override public ScannerResult call() throws Exception {
+ return server.scanNext(id, numberOfRows);
+ }
+ });
}
@Override
- public boolean scanClose(long id) throws ThriftHBaseException {
- return server.scanClose(id);
+ public ListenableFuture<Boolean> scanClose(final long id) {
+ return readService.submit(new Callable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ return server.scanClose(id);
+ }
+ });
+ }
+
+ /**
+ * If there is a 'regioninfo' value in the result, it would be in Writable
+ * serialized form. Add a thrift serialized HRegionInfo object for the
+ * non-Java objects.
+ */
+ private Result addThriftRegionInfoQualifierIfNeeded(Result tentativeResult) {
+ //TODO: Thrift has some problem serializing HRegionInfo. Since this method is only
+ // for C++ client, I temporarily disable it. After the problem is fixed we should
+ // remove the flag.
+ if (HConstants.DISABLE_THRIFT_REGION_INFO_QUALIFIER) {
+ return tentativeResult;
+ }
+
+ // Get the serialized HRegionInfo object
+ byte[] value = tentativeResult.searchValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER);
+ // If the value exists, then we need to do something, otherwise, we return
+ // the result untouched.
+ if (value != null && value.length > 0) {
+ try {
+ // Get the Writable-serialized HRegionInfo object.
+ HRegionInfo hri = (HRegionInfo) Writables.getWritable(value,
+ new HRegionInfo());
+ byte[] thriftSerializedHri =
+ Bytes.writeThriftBytes(hri, HRegionInfo.class);
+ // Create the KV with the thrift-serialized HRegionInfo
+ List<KeyValue> kvList = tentativeResult.getKvs();
+ KeyValue kv = new KeyValue(kvList.get(0).getRow(),
+ HConstants.CATALOG_FAMILY, HConstants.THRIFT_REGIONINFO_QUALIFIER,
+ thriftSerializedHri);
+ kvList.add(kv);
+ return new Result(kvList);
+ } catch (Exception e) {
+ // If failed, log the mistake and returns the original result
+ LOG.error("Thrift Serialization of the HRegionInfo object failed!");
+ e.printStackTrace();
+ }
+ }
+ return tentativeResult;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 6a9fec1..facdc1b 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.ipc.ThriftHRegionInterface;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.SizeAwareThriftHRegionServer;
import org.apache.hadoop.hbase.regionserver.ThriftHRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
@@ -439,8 +440,8 @@ public class MiniHBaseCluster {
return hbaseCluster.getRegionServer(serverNumber);
}
- public ThriftHRegionInterface.Sync getThriftRegionServer(int serverNumber) {
- return new ThriftHRegionServer(getRegionServer(serverNumber));
+ public ThriftHRegionInterface.Async getThriftRegionServer(int serverNumber) {
+ return new SizeAwareThriftHRegionServer(getRegionServer(serverNumber));
}
public List<HRegion> getRegions(byte[] tableName) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/test/java/org/apache/hadoop/hbase/regionserver/FailureInjectingThriftHRegionServer.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/FailureInjectingThriftHRegionServer.java b/src/test/java/org/apache/hadoop/hbase/regionserver/FailureInjectingThriftHRegionServer.java
index 2015343..21bc107 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/FailureInjectingThriftHRegionServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/FailureInjectingThriftHRegionServer.java
@@ -68,7 +68,7 @@ public class FailureInjectingThriftHRegionServer extends ThriftHRegionServer {
}
@Override
- public Result get(byte[] regionName, Get get) throws ThriftHBaseException {
+ public ListenableFuture<Result> get(byte[] regionName, Get get) {
switch (failureType) {
// Region server will throw RegionOverloadedException, wrapped in ThriftHBaseException
case REGIONOVERLOADEDEXCEPTION:
@@ -77,8 +77,8 @@ public class FailureInjectingThriftHRegionServer extends ThriftHRegionServer {
return super.get(regionName, get);
}
LOG.debug("Exception repeat count: " + repeatCount);
- throw new ThriftHBaseException(
- new RegionOverloadedException("GetAsync Test ROE", HConstants.DEFAULT_HBASE_CLIENT_PAUSE));
+ return Futures.immediateFailedFuture(new ThriftHBaseException(
+ new RegionOverloadedException("GetAsync Test ROE", HConstants.DEFAULT_HBASE_CLIENT_PAUSE)));
// Region server will disconnect the client, cause TTransportException on client side
case STOP:
@@ -101,7 +101,7 @@ public class FailureInjectingThriftHRegionServer extends ThriftHRegionServer {
return super.get(regionName, get);
}
LOG.debug("Exception repeat count: " + repeatCount);
- throw new ThriftHBaseException(getRetriableExceptions(repeatCount));
+ return Futures.immediateFailedFuture(new ThriftHBaseException(getRetriableExceptions(repeatCount)));
case DONOTRETRYEXCEPTION:
if (++repeatCount > repeats) {
@@ -109,8 +109,8 @@ public class FailureInjectingThriftHRegionServer extends ThriftHRegionServer {
return super.get(regionName, get);
}
LOG.debug("Exception repeat count: " + repeatCount);
- throw new ThriftHBaseException(
- new DoNotRetryIOException("GetAsync Test DoNotRetryIOE"));
+ return Futures.immediateFailedFuture( new ThriftHBaseException(
+ new DoNotRetryIOException("GetAsync Test DoNotRetryIOE")));
default:
return super.get(regionName, get);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b30de6e/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestHRegionInterfaceSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestHRegionInterfaceSimpleFunctions.java b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestHRegionInterfaceSimpleFunctions.java
index 9af0b66..8667a8e 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestHRegionInterfaceSimpleFunctions.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestHRegionInterfaceSimpleFunctions.java
@@ -73,7 +73,7 @@ public class TestHRegionInterfaceSimpleFunctions {
throws IOException, InterruptedException, ExecutionException, TimeoutException, ThriftHBaseException {
HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
- ThriftHRegionInterface.Sync thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
+ ThriftHRegionInterface.Async thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
TEST_UTIL.loadTable(table, FAMILY);
HServerInfo info = server.getHServerInfo();
@@ -151,7 +151,7 @@ public class TestHRegionInterfaceSimpleFunctions {
assertEquals(client.getHLogsList(false), server.getHLogsList(false));
// getReginoAssignment
- List<HRegionInfo> infos = thriftServer.getRegionsAssignment();
+ List<HRegionInfo> infos = thriftServer.getRegionsAssignment().get();
assertFalse(infos.isEmpty());
}
@@ -161,7 +161,7 @@ public class TestHRegionInterfaceSimpleFunctions {
@Test
public void testAtomicMutation() throws Exception {
HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
- ThriftHRegionInterface.Sync thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
+ ThriftHRegionInterface.Async thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
byte[] row = Bytes.toBytes("test-row");
byte[] invalidValue = Bytes.toBytes("test-row2");
@@ -169,18 +169,18 @@ public class TestHRegionInterfaceSimpleFunctions {
byte[] regionName = regionInfo.getRegionName();
Put put = new Put(row);
put.add(FAMILY, null, row);
- assertTrue(thriftServer.checkAndPut(regionName, row, FAMILY, null, null, put));
+ assertTrue(thriftServer.checkAndPut(regionName, row, FAMILY, null, null, put).get());
assertFalse(thriftServer.checkAndPut(
- regionInfo.getRegionName(), row, FAMILY, null, invalidValue, put));
+ regionInfo.getRegionName(), row, FAMILY, null, invalidValue, put).get());
Delete delete = new Delete(row);
delete.deleteFamily(FAMILY);
- assertFalse(thriftServer.checkAndDelete(regionName, row, FAMILY, null, invalidValue, delete));
- assertTrue(thriftServer.checkAndDelete(regionName, row, FAMILY, null, row, delete));
+ assertFalse(thriftServer.checkAndDelete(regionName, row, FAMILY, null, invalidValue, delete).get());
+ assertTrue(thriftServer.checkAndDelete(regionName, row, FAMILY, null, row, delete).get());
put = new Put(row);
put.add(FAMILY, null, Bytes.toBytes(1L));
- thriftServer.put(regionName, put);
- long result = thriftServer.incrementColumnValue(regionName, row, FAMILY, null, 100L, false);
+ thriftServer.put(regionName, put).get();
+ long result = thriftServer.incrementColumnValue(regionName, row, FAMILY, null, 100L, false).get();
assertEquals(101L, result);
}
@@ -190,13 +190,13 @@ public class TestHRegionInterfaceSimpleFunctions {
@Test
public void testQuorumConfigurationChanges() throws Exception {
HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
- ThriftHRegionInterface.Sync thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
+ ThriftHRegionInterface.Async thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
int threads = server.getQuorumReadThreadsMax() + 1;
long timeout = server.getQuorumReadTimeoutMillis() + 1;
- thriftServer.setNumHDFSQuorumReadThreads(threads);
- thriftServer.setHDFSQuorumReadTimeoutMillis(timeout);
+ thriftServer.setNumHDFSQuorumReadThreads(threads).get();
+ thriftServer.setHDFSQuorumReadTimeoutMillis(timeout).get();
assertEquals(threads, server.getQuorumReadThreadsMax());
assertEquals(timeout, server.getQuorumReadTimeoutMillis());
@@ -208,7 +208,7 @@ public class TestHRegionInterfaceSimpleFunctions {
@Test
public void testCloseRegion() throws Exception {
HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
- ThriftHRegionInterface.Sync thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
+ ThriftHRegionInterface.Async thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
HRegion[] region = server.getOnlineRegionsAsArray();
HRegionInfo regionInfo = region[0].getRegionInfo();
@@ -218,7 +218,7 @@ public class TestHRegionInterfaceSimpleFunctions {
String regionZNode = zkWrapper.getZNode(
zkWrapper.getRegionInTransitionZNode(), regionInfo.getEncodedName());
- thriftServer.closeRegion(regionInfo, true);
+ thriftServer.closeRegion(regionInfo, true).get();
byte[] data = zkWrapper.readZNode(regionZNode, new Stat());
RegionTransitionEventData rsData = new RegionTransitionEventData();
@@ -235,11 +235,11 @@ public class TestHRegionInterfaceSimpleFunctions {
*/
@Test
public void testStopRegionServer() throws Exception {
- ThriftHRegionInterface.Sync thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
- thriftServer.updateConfiguration();
+ ThriftHRegionInterface.Async thriftServer = TEST_UTIL.getHBaseCluster().getThriftRegionServer(0);
+ thriftServer.updateConfiguration().get();
String why = "test reason";
thriftServer.stop(why);
- assertEquals(thriftServer.getStopReason(), why);
+ assertEquals(thriftServer.getStopReason().get(), why);
}
}
[35/50] [abbrv] git commit: [master] Fix TestServerSideException
Posted by ec...@apache.org.
[master] Fix TestServerSideException
Summary: Fix TestServerSideException
Test Plan: mvn clean test -Dtest=TestServerSideException
Reviewers: aaiyer, manukranthk
Reviewed By: manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1410139
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43231 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/153336f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/153336f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/153336f1
Branch: refs/heads/0.89-fb
Commit: 153336f1c0e8bcd2f9fce2c56554c1724ab03d88
Parents: 4282d5d
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 30 20:52:55 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:24 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/thrift/swift/TestServerSideException.java | 2 +-
.../hbase/thrift/swift/TestServerSideExceptionDestructive.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/153336f1/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideException.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideException.java b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideException.java
index ef2fec9..3250686 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideException.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideException.java
@@ -73,7 +73,7 @@ public class TestServerSideException {
true);
conf.setClass(HConstants.THRIFT_REGION_SERVER_IMPL,
- FailureInjectingThriftHRegionServer.class, ThriftHRegionInterface.Sync.class);
+ FailureInjectingThriftHRegionServer.class, ThriftHRegionInterface.Async.class);
conf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
// Server will allow client to retry once when there is RegionOverloadedException
http://git-wip-us.apache.org/repos/asf/hbase/blob/153336f1/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideExceptionDestructive.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideExceptionDestructive.java b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideExceptionDestructive.java
index dcf17fa..4cc9435 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideExceptionDestructive.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestServerSideExceptionDestructive.java
@@ -56,7 +56,7 @@ public class TestServerSideExceptionDestructive {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setClass(HConstants.THRIFT_REGION_SERVER_IMPL,
- FailureInjectingThriftHRegionServer.class, ThriftHRegionInterface.Sync.class);
+ FailureInjectingThriftHRegionServer.class, ThriftHRegionInterface.Async.class);
conf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);