You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/11/30 17:58:20 UTC
[1/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE
during split transaction
Repository: hbase
Updated Branches:
refs/heads/branch-1 f1e2f077f -> bd87f4ebc
refs/heads/branch-1.4 302edf447 -> 70d9934e4
HBASE-21464 Splitting blocked with meta NSRE during split transaction
When looking up the locations of hbase:meta with useCache false, clear all previous
cache entries for it first
Fix Table reference leaks in MetaTableAccessor with try-with-resources
Signed-off-by: Allan Yang <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd87f4eb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd87f4eb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd87f4eb
Branch: refs/heads/branch-1
Commit: bd87f4ebcd49e9a3d100bba81fd6ab8868027c06
Parents: f1e2f07
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Nov 29 15:46:21 2018 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 30 09:57:54 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/MetaTableAccessor.java | 95 +++++++++-----------
.../hadoop/hbase/client/ConnectionManager.java | 10 ++-
2 files changed, 49 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd87f4eb/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 440f8c6..fd7a97b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -175,8 +175,7 @@ public class MetaTableAccessor {
* @return An {@link Table} for <code>hbase:meta</code>
* @throws IOException
*/
- static Table getMetaHTable(final Connection connection)
- throws IOException {
+ static Table getMetaHTable(final Connection connection) throws IOException {
// We used to pass whole CatalogTracker in here, now we just pass in Connection
if (connection == null) {
throw new NullPointerException("No connection");
@@ -248,11 +247,13 @@ public class MetaTableAccessor {
}
Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getMetaHTable(connection), get);
- RegionLocations locations = getRegionLocations(r);
- return locations == null
- ? null
- : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+ try (Table metaTable = getMetaHTable(connection)) {
+ Result r = get(metaTable, get);
+ RegionLocations locations = getRegionLocations(r);
+ return locations == null
+ ? null
+ : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+ }
}
/**
@@ -267,8 +268,10 @@ public class MetaTableAccessor {
byte[] row = getMetaKeyForRegion(regionInfo);
Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getMetaHTable(connection), get);
- return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+ try (Table metaTable = getMetaHTable(connection)) {
+ Result r = get(metaTable, get);
+ return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+ }
}
/** Returns the row key to use for this regionInfo */
@@ -300,7 +303,9 @@ public class MetaTableAccessor {
byte[] regionName) throws IOException {
Get get = new Get(regionName);
get.addFamily(HConstants.CATALOG_FAMILY);
- return get(getMetaHTable(connection), get);
+ try (Table metaTable = getMetaHTable(connection)) {
+ return get(metaTable, get);
+ }
}
/**
@@ -631,19 +636,19 @@ public class MetaTableAccessor {
scan.setCaching(caching);
}
scan.addFamily(HConstants.CATALOG_FAMILY);
- Table metaTable = getMetaHTable(connection);
- ResultScanner scanner = null;
- try {
- scanner = metaTable.getScanner(scan);
- Result data;
- while((data = scanner.next()) != null) {
- if (data.isEmpty()) continue;
- // Break if visit returns false.
- if (!visitor.visit(data)) break;
+ try (Table metaTable = getMetaHTable(connection)) {
+ try (ResultScanner scanner = metaTable.getScanner(scan)) {
+ Result data;
+ while ((data = scanner.next()) != null) {
+ if (data.isEmpty()) {
+ continue;
+ }
+ // Break if visit returns false.
+ if (!visitor.visit(data)) {
+ break;
+ }
+ }
}
- } finally {
- if (scanner != null) scanner.close();
- metaTable.close();
}
}
@@ -1020,7 +1025,9 @@ public class MetaTableAccessor {
*/
static void putToMetaTable(final Connection connection, final Put p)
throws IOException {
- put(getMetaHTable(connection), p);
+ try (Table metaTable = getMetaHTable(connection)) {
+ put(metaTable, p);
+ }
}
/**
@@ -1044,11 +1051,8 @@ public class MetaTableAccessor {
*/
public static void putsToMetaTable(final Connection connection, final List<Put> ps)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.put(ps);
- } finally {
- t.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.put(ps);
}
}
@@ -1073,11 +1077,8 @@ public class MetaTableAccessor {
*/
public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.delete(deletes);
- } finally {
- t.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.delete(deletes);
}
}
@@ -1116,15 +1117,12 @@ public class MetaTableAccessor {
public static void mutateMetaTable(final Connection connection,
final List<Mutation> mutations)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.batch(mutations);
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.batch(mutations);
} catch (InterruptedException e) {
InterruptedIOException ie = new InterruptedIOException(e.getMessage());
ie.initCause(e);
throw ie;
- } finally {
- t.close();
}
}
@@ -1188,11 +1186,8 @@ public class MetaTableAccessor {
*/
public static void addRegionToMeta(Connection connection, HRegionInfo regionInfo,
HRegionInfo splitA, HRegionInfo splitB) throws IOException {
- Table meta = getMetaHTable(connection);
- try {
- addRegionToMeta(meta, regionInfo, splitA, splitB);
- } finally {
- meta.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ addRegionToMeta(metaTable, regionInfo, splitA, splitB);
}
}
@@ -1269,8 +1264,7 @@ public class MetaTableAccessor {
HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication,
long masterSystemTime)
throws IOException {
- Table meta = getMetaHTable(connection);
- try {
+ try (Table metaTable = getMetaHTable(connection)) {
HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
// use the maximum of what master passed us vs local time.
@@ -1298,9 +1292,7 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
+ HConstants.DELIMITER);
- multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
- } finally {
- meta.close();
+ multiMutate(metaTable, tableRow, putOfMerged, deleteA, deleteB);
}
}
@@ -1318,8 +1310,7 @@ public class MetaTableAccessor {
public static void splitRegion(final Connection connection,
HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
ServerName sn, int regionReplication) throws IOException {
- Table meta = getMetaHTable(connection);
- try {
+ try (Table metaTable = getMetaHTable(connection)) {
HRegionInfo copyOfParent = new HRegionInfo(parent);
copyOfParent.setOffline(true);
copyOfParent.setSplit(true);
@@ -1343,9 +1334,7 @@ public class MetaTableAccessor {
}
byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
- multiMutate(meta, tableRow, putParent, putA, putB);
- } finally {
- meta.close();
+ multiMutate(metaTable, tableRow, putParent, putA, putB);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bd87f4eb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 7cf09c2..58f2c52 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -1236,10 +1236,9 @@ class ConnectionManager {
// HBASE-10785: We cache the location of the META itself, so that we are not overloading
// zookeeper with one request for every region lookup. We cache the META with empty row
// key in MetaCache.
- byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta
RegionLocations locations = null;
if (useCache) {
- locations = getCachedLocation(tableName, metaCacheKey);
+ locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
if (locations != null && locations.getRegionLocation(replicaId) != null) {
return locations;
}
@@ -1250,10 +1249,15 @@ class ConnectionManager {
// Check the cache again for a hit in case some other thread made the
// same query while we were waiting on the lock.
if (useCache) {
- locations = getCachedLocation(tableName, metaCacheKey);
+ locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
if (locations != null && locations.getRegionLocation(replicaId) != null) {
return locations;
}
+ } else {
+ // Don't keep stale entries in cache when relocating meta. Cache state should
+ // reflect whatever is up in zookeeper (which could be the case where no region
+ // is deployed yet).
+ clearRegionCache(tableName);
}
// Look up from zookeeper
[2/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE
during split transaction
Posted by ap...@apache.org.
HBASE-21464 Splitting blocked with meta NSRE during split transaction
When looking up the locations of hbase:meta with useCache false, clear all previous
cache entries for it first
Fix Table reference leaks in MetaTableAccessor with try-with-resources
Signed-off-by: Allan Yang <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70d9934e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70d9934e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70d9934e
Branch: refs/heads/branch-1.4
Commit: 70d9934e40785d13c9220d4b3ab0c3b31ca59f8c
Parents: 302edf4
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Nov 29 15:46:21 2018 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 30 09:58:08 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/MetaTableAccessor.java | 95 +++++++++-----------
.../hadoop/hbase/client/ConnectionManager.java | 10 ++-
2 files changed, 49 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/70d9934e/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 440f8c6..fd7a97b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -175,8 +175,7 @@ public class MetaTableAccessor {
* @return An {@link Table} for <code>hbase:meta</code>
* @throws IOException
*/
- static Table getMetaHTable(final Connection connection)
- throws IOException {
+ static Table getMetaHTable(final Connection connection) throws IOException {
// We used to pass whole CatalogTracker in here, now we just pass in Connection
if (connection == null) {
throw new NullPointerException("No connection");
@@ -248,11 +247,13 @@ public class MetaTableAccessor {
}
Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getMetaHTable(connection), get);
- RegionLocations locations = getRegionLocations(r);
- return locations == null
- ? null
- : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+ try (Table metaTable = getMetaHTable(connection)) {
+ Result r = get(metaTable, get);
+ RegionLocations locations = getRegionLocations(r);
+ return locations == null
+ ? null
+ : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+ }
}
/**
@@ -267,8 +268,10 @@ public class MetaTableAccessor {
byte[] row = getMetaKeyForRegion(regionInfo);
Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getMetaHTable(connection), get);
- return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+ try (Table metaTable = getMetaHTable(connection)) {
+ Result r = get(metaTable, get);
+ return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+ }
}
/** Returns the row key to use for this regionInfo */
@@ -300,7 +303,9 @@ public class MetaTableAccessor {
byte[] regionName) throws IOException {
Get get = new Get(regionName);
get.addFamily(HConstants.CATALOG_FAMILY);
- return get(getMetaHTable(connection), get);
+ try (Table metaTable = getMetaHTable(connection)) {
+ return get(metaTable, get);
+ }
}
/**
@@ -631,19 +636,19 @@ public class MetaTableAccessor {
scan.setCaching(caching);
}
scan.addFamily(HConstants.CATALOG_FAMILY);
- Table metaTable = getMetaHTable(connection);
- ResultScanner scanner = null;
- try {
- scanner = metaTable.getScanner(scan);
- Result data;
- while((data = scanner.next()) != null) {
- if (data.isEmpty()) continue;
- // Break if visit returns false.
- if (!visitor.visit(data)) break;
+ try (Table metaTable = getMetaHTable(connection)) {
+ try (ResultScanner scanner = metaTable.getScanner(scan)) {
+ Result data;
+ while ((data = scanner.next()) != null) {
+ if (data.isEmpty()) {
+ continue;
+ }
+ // Break if visit returns false.
+ if (!visitor.visit(data)) {
+ break;
+ }
+ }
}
- } finally {
- if (scanner != null) scanner.close();
- metaTable.close();
}
}
@@ -1020,7 +1025,9 @@ public class MetaTableAccessor {
*/
static void putToMetaTable(final Connection connection, final Put p)
throws IOException {
- put(getMetaHTable(connection), p);
+ try (Table metaTable = getMetaHTable(connection)) {
+ put(metaTable, p);
+ }
}
/**
@@ -1044,11 +1051,8 @@ public class MetaTableAccessor {
*/
public static void putsToMetaTable(final Connection connection, final List<Put> ps)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.put(ps);
- } finally {
- t.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.put(ps);
}
}
@@ -1073,11 +1077,8 @@ public class MetaTableAccessor {
*/
public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.delete(deletes);
- } finally {
- t.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.delete(deletes);
}
}
@@ -1116,15 +1117,12 @@ public class MetaTableAccessor {
public static void mutateMetaTable(final Connection connection,
final List<Mutation> mutations)
throws IOException {
- Table t = getMetaHTable(connection);
- try {
- t.batch(mutations);
+ try (Table metaTable = getMetaHTable(connection)) {
+ metaTable.batch(mutations);
} catch (InterruptedException e) {
InterruptedIOException ie = new InterruptedIOException(e.getMessage());
ie.initCause(e);
throw ie;
- } finally {
- t.close();
}
}
@@ -1188,11 +1186,8 @@ public class MetaTableAccessor {
*/
public static void addRegionToMeta(Connection connection, HRegionInfo regionInfo,
HRegionInfo splitA, HRegionInfo splitB) throws IOException {
- Table meta = getMetaHTable(connection);
- try {
- addRegionToMeta(meta, regionInfo, splitA, splitB);
- } finally {
- meta.close();
+ try (Table metaTable = getMetaHTable(connection)) {
+ addRegionToMeta(metaTable, regionInfo, splitA, splitB);
}
}
@@ -1269,8 +1264,7 @@ public class MetaTableAccessor {
HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication,
long masterSystemTime)
throws IOException {
- Table meta = getMetaHTable(connection);
- try {
+ try (Table metaTable = getMetaHTable(connection)) {
HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
// use the maximum of what master passed us vs local time.
@@ -1298,9 +1292,7 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
+ HConstants.DELIMITER);
- multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
- } finally {
- meta.close();
+ multiMutate(metaTable, tableRow, putOfMerged, deleteA, deleteB);
}
}
@@ -1318,8 +1310,7 @@ public class MetaTableAccessor {
public static void splitRegion(final Connection connection,
HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
ServerName sn, int regionReplication) throws IOException {
- Table meta = getMetaHTable(connection);
- try {
+ try (Table metaTable = getMetaHTable(connection)) {
HRegionInfo copyOfParent = new HRegionInfo(parent);
copyOfParent.setOffline(true);
copyOfParent.setSplit(true);
@@ -1343,9 +1334,7 @@ public class MetaTableAccessor {
}
byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
- multiMutate(meta, tableRow, putParent, putA, putB);
- } finally {
- meta.close();
+ multiMutate(metaTable, tableRow, putParent, putA, putB);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/70d9934e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 7cf09c2..58f2c52 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -1236,10 +1236,9 @@ class ConnectionManager {
// HBASE-10785: We cache the location of the META itself, so that we are not overloading
// zookeeper with one request for every region lookup. We cache the META with empty row
// key in MetaCache.
- byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta
RegionLocations locations = null;
if (useCache) {
- locations = getCachedLocation(tableName, metaCacheKey);
+ locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
if (locations != null && locations.getRegionLocation(replicaId) != null) {
return locations;
}
@@ -1250,10 +1249,15 @@ class ConnectionManager {
// Check the cache again for a hit in case some other thread made the
// same query while we were waiting on the lock.
if (useCache) {
- locations = getCachedLocation(tableName, metaCacheKey);
+ locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
if (locations != null && locations.getRegionLocation(replicaId) != null) {
return locations;
}
+ } else {
+ // Don't keep stale entries in cache when relocating meta. Cache state should
+ // reflect whatever is up in zookeeper (which could be the case where no region
+ // is deployed yet).
+ clearRegionCache(tableName);
}
// Look up from zookeeper