You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/12/23 21:30:56 UTC
[1/2] hbase git commit: HBASE-12736 Let MetaScanner recycle a given
Connection (Jurriaan Mous)
Repository: hbase
Updated Branches:
refs/heads/branch-1 59f93e238 -> 4cb5d15b6
HBASE-12736 Let MetaScanner recycle a given Connection (Jurriaan Mous)
Signed-off-by: stack <st...@apache.org>
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/611ee0eb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/611ee0eb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/611ee0eb
Branch: refs/heads/branch-1
Commit: 611ee0eb7214ee35b912cea677864db13e6c8504
Parents: 59f93e2
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sat Dec 20 15:07:54 2014 +0100
Committer: stack <st...@apache.org>
Committed: Tue Dec 23 10:59:18 2014 -0800
----------------------------------------------------------------------
.../hadoop/hbase/client/ConnectionManager.java | 7 +--
.../apache/hadoop/hbase/client/HBaseAdmin.java | 4 +-
.../org/apache/hadoop/hbase/client/HTable.java | 2 +-
.../apache/hadoop/hbase/client/MetaScanner.java | 59 +++++++++-----------
.../hbase/client/TestClientNoCluster.java | 6 +-
.../hadoop/hbase/rest/RegionsResource.java | 8 ++-
.../hadoop/hbase/master/CatalogJanitor.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hadoop/hbase/client/TestMetaScanner.java | 29 +++++-----
.../hbase/master/TestRegionPlacement.java | 11 ++--
.../hadoop/hbase/master/TestRestartCluster.java | 17 +++++-
.../TestEndToEndSplitTransaction.java | 19 +++++--
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 12 ++--
14 files changed, 102 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 895e486..88e045b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -906,7 +906,7 @@ class ConnectionManager {
return true;
}
};
- MetaScanner.metaScan(conf, this, visitor, tableName);
+ MetaScanner.metaScan(this, visitor, tableName);
return available.get() && (regionCount.get() > 0);
}
@@ -951,7 +951,7 @@ class ConnectionManager {
return true;
}
};
- MetaScanner.metaScan(conf, this, visitor, tableName);
+ MetaScanner.metaScan(this, visitor, tableName);
// +1 needs to be added so that the empty start row is also taken into account
return available.get() && (regionCount.get() == splitKeys.length + 1);
}
@@ -993,8 +993,7 @@ class ConnectionManager {
@Override
public List<HRegionLocation> locateRegions(final TableName tableName,
final boolean useCache, final boolean offlined) throws IOException {
- NavigableMap<HRegionInfo, ServerName> regions =
- MetaScanner.allTableRegions(conf, this, tableName);
+ NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(this, tableName);
final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
for (HRegionInfo regionInfo : regions.keySet()) {
RegionLocations list = locateRegion(tableName, regionInfo.getStartKey(), useCache, true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 40ff168..8cd3037 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -576,7 +576,7 @@ public class HBaseAdmin implements Admin {
return true;
}
};
- MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
+ MetaScanner.metaScan(connection, visitor, desc.getTableName());
if (actualRegCount.get() < numRegs) {
if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
throw new RegionOfflineException("Only " + actualRegCount.get() +
@@ -2223,7 +2223,7 @@ public class HBaseAdmin implements Admin {
}
};
- MetaScanner.metaScan(conf, connection, visitor, null);
+ MetaScanner.metaScan(connection, visitor, null);
pair = result.get();
}
return pair;
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 146cf80..c141b29 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -654,7 +654,7 @@ public class HTable implements HTableInterface, RegionLocator {
@Deprecated
public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOException {
// TODO: Odd that this returns a Map of HRI to SN whereas getRegionLocator, singular, returns an HRegionLocation.
- return MetaScanner.allTableRegions(getConfiguration(), this.connection, getName());
+ return MetaScanner.allTableRegions(this.connection, getName());
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index fdffec4..8acf22a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -63,32 +63,30 @@ public class MetaScanner {
* start row value as table name.
*
* <p>Visible for testing. Use {@link
- * #metaScan(Configuration, Connection, MetaScannerVisitor, TableName)} instead.
+ * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
*
- * @param configuration conf
* @param visitor A custom visitor
* @throws IOException e
*/
@VisibleForTesting // Do not use. Used by tests only and hbck.
- public static void metaScan(Configuration configuration, MetaScannerVisitor visitor)
- throws IOException {
- metaScan(configuration, visitor, null, null, Integer.MAX_VALUE);
+ public static void metaScan(Connection connection,
+ MetaScannerVisitor visitor) throws IOException {
+ metaScan(connection, visitor, null, null, Integer.MAX_VALUE);
}
/**
* Scans the meta table and calls a visitor on each RowResult. Uses a table
* name to locate meta regions.
*
- * @param configuration config
* @param connection connection to use internally (null to use a new instance)
* @param visitor visitor object
* @param userTableName User table name in meta table to start scan at. Pass
* null if not interested in a particular table.
* @throws IOException e
*/
- public static void metaScan(Configuration configuration, Connection connection,
+ public static void metaScan(Connection connection,
MetaScannerVisitor visitor, TableName userTableName) throws IOException {
- metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
+ metaScan(connection, visitor, userTableName, null, Integer.MAX_VALUE,
TableName.META_TABLE_NAME);
}
@@ -98,9 +96,9 @@ public class MetaScanner {
* <code>rowLimit</code> of rows.
*
* <p>Visible for testing. Use {@link
- * #metaScan(Configuration, Connection, MetaScannerVisitor, TableName)} instead.
+ * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
*
- * @param configuration HBase configuration.
+ * @param connection to scan on
* @param visitor Visitor object.
* @param userTableName User table name in meta table to start scan at. Pass
* null if not interested in a particular table.
@@ -111,11 +109,12 @@ public class MetaScanner {
* @throws IOException e
*/
@VisibleForTesting // Do not use. Used by Master but by a method that is used testing.
- public static void metaScan(Configuration configuration,
+ public static void metaScan(Connection connection,
MetaScannerVisitor visitor, TableName userTableName, byte[] row,
int rowLimit)
throws IOException {
- metaScan(configuration, null, visitor, userTableName, row, rowLimit, TableName.META_TABLE_NAME);
+ metaScan(connection, visitor, userTableName, row, rowLimit, TableName
+ .META_TABLE_NAME);
}
/**
@@ -123,7 +122,6 @@ public class MetaScanner {
* name and a row name to locate meta regions. And it only scans at most
* <code>rowLimit</code> of rows.
*
- * @param configuration HBase configuration.
* @param connection connection to use internally (null to use a new instance)
* @param visitor Visitor object. Closes the visitor before returning.
* @param tableName User table name in meta table to start scan at. Pass
@@ -135,17 +133,11 @@ public class MetaScanner {
* @param metaTableName Meta table to scan, root or meta.
* @throws IOException e
*/
- static void metaScan(Configuration configuration, Connection connection,
+ static void metaScan(Connection connection,
final MetaScannerVisitor visitor, final TableName tableName,
final byte[] row, final int rowLimit, final TableName metaTableName)
throws IOException {
- boolean closeConnection = false;
- if (connection == null) {
- connection = ConnectionFactory.createConnection(configuration);
- closeConnection = true;
- }
-
int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
// Calculate startrow for scan.
byte[] startRow;
@@ -179,8 +171,9 @@ public class MetaScanner {
HConstants.ZEROES, false);
}
final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
- int scannerCaching = configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
- HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
+ int scannerCaching = connection.getConfiguration()
+ .getInt(HConstants.HBASE_META_SCANNER_CACHING,
+ HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
if (rowUpperLimit <= scannerCaching) {
scan.setSmall(true);
}
@@ -211,9 +204,6 @@ public class MetaScanner {
LOG.debug("Got exception in closing the meta scanner visitor", t);
}
}
- if (closeConnection) {
- if (connection != null) connection.close();
- }
}
}
@@ -246,14 +236,16 @@ public class MetaScanner {
/**
* Lists all of the regions currently in META.
- * @param conf
+ * @param conf configuration
+ * @param connection to connect with
* @param offlined True if we are to include offlined regions, false and we'll
* leave out offlined regions from returned list.
* @return List of all user-space regions.
* @throws IOException
*/
@VisibleForTesting // And for hbck.
- public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
+ public static List<HRegionInfo> listAllRegions(Configuration conf, Connection connection,
+ final boolean offlined)
throws IOException {
final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
@@ -276,7 +268,7 @@ public class MetaScanner {
return true;
}
};
- metaScan(conf, visitor);
+ metaScan(connection, visitor);
return regions;
}
@@ -287,23 +279,22 @@ public class MetaScanner {
* leave out offlined regions from returned list.
* @return Map of all user-space regions to servers
* @throws IOException
- * @deprecated Use {@link #allTableRegions(Configuration, Connection, TableName)} instead
+ * @deprecated Use {@link #allTableRegions(Connection, TableName)} instead
*/
@Deprecated
public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
Connection connection, final TableName tableName, boolean offlined) throws IOException {
- return allTableRegions(conf, connection, tableName);
+ return allTableRegions(connection, tableName);
}
/**
* Lists all of the table regions currently in META.
- * @param conf
* @param connection
* @param tableName
* @return Map of all user-space regions to servers
* @throws IOException
*/
- public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
+ public static NavigableMap<HRegionInfo, ServerName> allTableRegions(
Connection connection, final TableName tableName) throws IOException {
final NavigableMap<HRegionInfo, ServerName> regions =
new TreeMap<HRegionInfo, ServerName>();
@@ -321,7 +312,7 @@ public class MetaScanner {
return true;
}
};
- metaScan(conf, connection, visitor, tableName);
+ metaScan(connection, visitor, tableName);
return regions;
}
@@ -340,7 +331,7 @@ public class MetaScanner {
return true;
}
};
- metaScan(conf, connection, visitor, tableName);
+ metaScan(connection, visitor, tableName);
return regions;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
index 9582405..b1ef5b9 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
@@ -175,7 +175,7 @@ public class TestClientNoCluster extends Configured implements Tool {
* @throws IOException
*/
@Test
- public void testRocTimeout() throws IOException {
+ public void testRpcTimeout() throws IOException {
Configuration localConfig = HBaseConfiguration.create(this.conf);
// This override mocks up our exists/get call to throw a RegionServerStoppedException.
localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
@@ -209,7 +209,9 @@ public class TestClientNoCluster extends Configured implements Tool {
public void testDoNotRetryMetaScanner() throws IOException {
this.conf.set("hbase.client.connection.impl",
RegionServerStoppedOnScannerOpenConnection.class.getName());
- MetaScanner.metaScan(this.conf, null);
+ try (Connection connection = ConnectionFactory.createConnection(conf)) {
+ MetaScanner.metaScan(connection, null);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index 1ecb7c6..001c6b5 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.MetaScanner;
import org.apache.hadoop.hbase.rest.model.TableInfoModel;
import org.apache.hadoop.hbase.rest.model.TableRegionModel;
@@ -76,8 +78,10 @@ public class RegionsResource extends ResourceBase {
try {
TableName tableName = TableName.valueOf(tableResource.getName());
TableInfoModel model = new TableInfoModel(tableName.getNameAsString());
- Map<HRegionInfo,ServerName> regions = MetaScanner.allTableRegions(
- servlet.getConfiguration(), null, tableName);
+
+ Connection connection = ConnectionFactory.createConnection(servlet.getConfiguration());
+ Map<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(connection, tableName);
+ connection.close();
for (Map.Entry<HRegionInfo,ServerName> e: regions.entrySet()) {
HRegionInfo hri = e.getKey();
ServerName addr = e.getValue();
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 25c405c..9f71b90 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -166,7 +166,7 @@ public class CatalogJanitor extends Chore {
// Run full scan of hbase:meta catalog table passing in our custom visitor with
// the start row
- MetaScanner.metaScan(server.getConfiguration(), this.connection, visitor, tableName);
+ MetaScanner.metaScan(this.connection, visitor, tableName);
return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
count.get(), mergedRegions, splitParents);
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a85c2e7..4a33bcd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1649,7 +1649,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
};
- MetaScanner.metaScan(conf, visitor, tableName, rowKey, 1);
+ MetaScanner.metaScan(clusterConnection, visitor, tableName, rowKey, 1);
return result.get();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 7d50113..e08eeea 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -620,7 +620,7 @@ public class HBaseFsck extends Configured {
public void checkRegionBoundaries() {
try {
ByteArrayComparator comparator = new ByteArrayComparator();
- List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
+ List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
final RegionBoundariesInformation currentRegionBoundariesInformation =
new RegionBoundariesInformation();
Path hbaseRoot = FSUtils.getRootDir(getConf());
@@ -2984,7 +2984,7 @@ public class HBaseFsck extends Configured {
};
if (!checkMetaOnly) {
// Scan hbase:meta to pick up user regions
- MetaScanner.metaScan(getConf(), visitor);
+ MetaScanner.metaScan(connection, visitor);
}
errors.print("");
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
index c5e0570..332b990 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
@@ -53,9 +53,11 @@ import org.junit.experimental.categories.Category;
public class TestMetaScanner {
final Log LOG = LogFactory.getLog(getClass());
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Connection connection;
public void setUp() throws Exception {
TEST_UTIL.startMiniCluster(1);
+ this.connection = TEST_UTIL.getConnection();
}
@After
@@ -66,13 +68,13 @@ public class TestMetaScanner {
@Test
public void testMetaScanner() throws Exception {
LOG.info("Starting testMetaScanner");
+
setUp();
- final TableName TABLENAME =
- TableName.valueOf("testMetaScanner");
+ final TableName TABLENAME = TableName.valueOf("testMetaScanner");
final byte[] FAMILY = Bytes.toBytes("family");
TEST_UTIL.createTable(TABLENAME, FAMILY);
Configuration conf = TEST_UTIL.getConfiguration();
- HTable table = new HTable(conf, TABLENAME);
+ HTable table = (HTable) connection.getTable(TABLENAME);
TEST_UTIL.createMultiRegions(conf, table, FAMILY,
new byte[][]{
HConstants.EMPTY_START_ROW,
@@ -86,28 +88,28 @@ public class TestMetaScanner {
doReturn(true).when(visitor).processRow((Result)anyObject());
// Scanning the entire table should give us three rows
- MetaScanner.metaScan(conf, null, visitor, TABLENAME);
+ MetaScanner.metaScan(connection, visitor, TABLENAME);
verify(visitor, times(3)).processRow((Result)anyObject());
// Scanning the table with a specified empty start row should also
// give us three hbase:meta rows
reset(visitor);
doReturn(true).when(visitor).processRow((Result)anyObject());
- MetaScanner.metaScan(conf, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
+ MetaScanner.metaScan(connection, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
verify(visitor, times(3)).processRow((Result)anyObject());
// Scanning the table starting in the middle should give us two rows:
// region_a and region_b
reset(visitor);
doReturn(true).when(visitor).processRow((Result)anyObject());
- MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
+ MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
verify(visitor, times(2)).processRow((Result)anyObject());
// Scanning with a limit of 1 should only give us one row
reset(visitor);
- doReturn(true).when(visitor).processRow((Result)anyObject());
- MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
- verify(visitor, times(1)).processRow((Result)anyObject());
+ doReturn(true).when(visitor).processRow((Result) anyObject());
+ MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
+ verify(visitor, times(1)).processRow((Result) anyObject());
table.close();
}
@@ -134,8 +136,8 @@ public class TestMetaScanner {
public void run() {
while (!isStopped()) {
try {
- List<HRegionInfo> regions = MetaScanner.listAllRegions(
- TEST_UTIL.getConfiguration(), false);
+ List<HRegionInfo> regions = MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(),
+ connection, false);
//select a random region
HRegionInfo parent = regions.get(random.nextInt(regions.size()));
@@ -166,7 +168,7 @@ public class TestMetaScanner {
Bytes.toBytes(midKey),
end);
- MetaTableAccessor.splitRegion(TEST_UTIL.getHBaseAdmin().getConnection(),
+ MetaTableAccessor.splitRegion(connection,
parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));
Threads.sleep(random.nextInt(200));
@@ -189,7 +191,7 @@ public class TestMetaScanner {
while(!isStopped()) {
try {
NavigableMap<HRegionInfo, ServerName> regions =
- MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), null, TABLENAME);
+ MetaScanner.allTableRegions(connection, TABLENAME);
LOG.info("-------");
byte[] lastEndKey = HConstants.EMPTY_START_ROW;
@@ -242,4 +244,3 @@ public class TestMetaScanner {
}
}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
index ccb809d..0899785 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
@@ -48,6 +48,8 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.MetaScanner;
@@ -73,6 +75,7 @@ public class TestRegionPlacement {
final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static int SLAVES = 10;
+ private static Connection CONNECTION;
private static Admin admin;
private static RegionPlacementMaintainer rp;
private static Position[] positions = Position.values();
@@ -89,7 +92,8 @@ public class TestRegionPlacement {
FavoredNodeLoadBalancer.class, LoadBalancer.class);
conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
TEST_UTIL.startMiniCluster(SLAVES);
- admin = new HBaseAdmin(conf);
+ CONNECTION = TEST_UTIL.getConnection();
+ admin = CONNECTION.getAdmin();
rp = new RegionPlacementMaintainer(conf);
}
@@ -522,7 +526,7 @@ public class TestRegionPlacement {
@Override
public void close() throws IOException {}
};
- MetaScanner.metaScan(TEST_UTIL.getConfiguration(), visitor);
+ MetaScanner.metaScan(CONNECTION, visitor);
LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
totalRegionNum.intValue() + " regions running on the primary" +
" region servers" );
@@ -549,8 +553,7 @@ public class TestRegionPlacement {
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, splitKeys);
- @SuppressWarnings("deprecation")
- HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+ HTable ht = (HTable) CONNECTION.getTable(tableName);
@SuppressWarnings("deprecation")
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
assertEquals("Tried to create " + expectedRegions + " regions "
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 9565e53..504d98d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -28,7 +28,15 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.MetaScanner;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -92,6 +100,8 @@ public class TestRestartCluster {
@Test (timeout=300000)
public void testClusterRestart() throws Exception {
UTIL.startMiniCluster(3);
+ Connection connection = UTIL.getConnection();
+
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
@@ -104,7 +114,7 @@ public class TestRestartCluster {
}
List<HRegionInfo> allRegions =
- MetaScanner.listAllRegions(UTIL.getConfiguration(), true);
+ MetaScanner.listAllRegions(UTIL.getConfiguration(), connection, true);
assertEquals(4, allRegions.size());
LOG.info("\n\nShutting down cluster");
@@ -119,7 +129,8 @@ public class TestRestartCluster {
// Need to use a new 'Configuration' so we make a new HConnection.
// Otherwise we're reusing an HConnection that has gone stale because
// the shutdown of the cluster also called shut of the connection.
- allRegions = MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), true);
+ allRegions =
+ MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), connection, true);
assertEquals(4, allRegions.size());
LOG.info("\n\nWaiting for tables to be available");
for(TableName TABLE: TABLES) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index b970ca0..f762dfc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -44,6 +44,10 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -231,6 +235,7 @@ public class TestEndToEndSplitTransaction {
}
static class RegionSplitter extends Thread {
+ final Connection connection;
Throwable ex;
Table table;
TableName tableName;
@@ -244,6 +249,7 @@ public class TestEndToEndSplitTransaction {
this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
admin = TEST_UTIL.getHBaseAdmin();
rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+ connection = TEST_UTIL.getConnection();
}
@Override
@@ -251,8 +257,8 @@ public class TestEndToEndSplitTransaction {
try {
Random random = new Random();
for (int i= 0; i< 5; i++) {
- NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
- tableName);
+ NavigableMap<HRegionInfo, ServerName> regions =
+ MetaScanner.allTableRegions(connection, tableName);
if (regions.size() == 0) {
continue;
}
@@ -309,27 +315,30 @@ public class TestEndToEndSplitTransaction {
* Checks regions using MetaScanner, MetaTableAccessor and HTable methods
*/
static class RegionChecker extends Chore {
+ Connection connection;
Configuration conf;
TableName tableName;
Throwable ex;
- RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) {
+ RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) throws IOException {
super("RegionChecker", 10, stopper);
this.conf = conf;
this.tableName = tableName;
this.setDaemon(true);
+
+ this.connection = ConnectionFactory.createConnection(conf);
}
/** verify region boundaries obtained from MetaScanner */
void verifyRegionsUsingMetaScanner() throws Exception {
//MetaScanner.allTableRegions()
- NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
+ NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(connection,
tableName);
verifyTableRegions(regions.keySet());
//MetaScanner.listAllRegions()
- List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
+ List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, connection, false);
verifyTableRegions(Sets.newTreeSet(regionList));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/611ee0eb/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index b61a399..fb2cb1b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -303,7 +304,7 @@ public class TestHBaseFsck {
dumpMeta(htd.getTableName());
Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
- HConnection conn = (HConnection) ConnectionFactory.createConnection(conf);
+ ClusterConnection conn = (ClusterConnection) ConnectionFactory.createConnection(conf);
for (Entry<HRegionInfo, ServerName> e: hris.entrySet()) {
HRegionInfo hri = e.getKey();
ServerName hsa = e.getValue();
@@ -2015,15 +2016,15 @@ public class TestHBaseFsck {
*/
@Test (timeout=180000)
public void testMissingRegionInfoQualifier() throws Exception {
- TableName table =
- TableName.valueOf("testMissingRegionInfoQualifier");
+ Connection connection = ConnectionFactory.createConnection(conf);
+ TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
try {
setupTable(table);
// Mess it up by removing the RegionInfo for one region.
final List<Delete> deletes = new LinkedList<Delete>();
- Table meta = new HTable(conf, TableName.META_TABLE_NAME);
- MetaScanner.metaScan(conf, new MetaScanner.MetaScannerVisitor() {
+ Table meta = connection.getTable(TableName.META_TABLE_NAME);
+ MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
@Override
public boolean processRow(Result rowResult) throws IOException {
@@ -2060,6 +2061,7 @@ public class TestHBaseFsck {
} finally {
deleteTable(table);
}
+ connection.close();
}
[2/2] hbase git commit: HBASE-12739 Avoid too large identifier of
ZooKeeperWatcher (Liu Shaohui)
Posted by st...@apache.org.
HBASE-12739 Avoid too large identifier of ZooKeeperWatcher (Liu Shaohui)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4cb5d15b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4cb5d15b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4cb5d15b
Branch: refs/heads/branch-1
Commit: 4cb5d15b609dd195df09381f4660a0e0dffebe27
Parents: 611ee0e
Author: stack <st...@apache.org>
Authored: Tue Dec 23 12:20:30 2014 -0800
Committer: stack <st...@apache.org>
Committed: Tue Dec 23 12:21:05 2014 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4cb5d15b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index af879d7e..407d37b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -55,6 +55,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
// Identifier for this watcher (for logging only). It is made of the prefix
// passed on construction and the zookeeper sessionid.
+ private String prefix;
private String identifier;
// zookeeper quorum
@@ -156,9 +157,10 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
this.constructorCaller = e;
}
this.quorum = ZKConfig.getZKQuorumServersString(conf);
+ this.prefix = identifier;
// Identifier will get the sessionid appended later below down when we
// handle the syncconnect event.
- this.identifier = identifier;
+ this.identifier = identifier + "0x0";
this.abortable = abortable;
setNodeNames(conf);
this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, identifier);
@@ -389,7 +391,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
this.constructorCaller);
throw new NullPointerException("ZK is null");
}
- this.identifier = this.identifier + "-0x" +
+ this.identifier = this.prefix + "-0x" +
Long.toHexString(this.recoverableZooKeeper.getSessionId());
// Update our identifier. Otherwise ignore.
LOG.debug(this.identifier + " connected");