You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/11/15 05:47:54 UTC
hbase git commit: HBASE-19245 MultiTableInputFormatBase#getSplits
creates a Connection per Table
Repository: hbase
Updated Branches:
refs/heads/master 0ac18a812 -> 797670b12
HBASE-19245 MultiTableInputFormatBase#getSplits creates a Connection per Table
We make one Connection only instead of a Connection per table (Change is
just moving one line it involves right-shifting body of the function)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/797670b1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/797670b1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/797670b1
Branch: refs/heads/master
Commit: 797670b129526697eefb2578fedb2b272c65f52e
Parents: 0ac18a8
Author: Michael Stack <st...@apache.org>
Authored: Mon Nov 13 11:42:10 2017 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Tue Nov 14 21:47:35 2017 -0800
----------------------------------------------------------------------
.../mapreduce/MultiTableInputFormatBase.java | 108 ++++++++++---------
1 file changed, 55 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/797670b1/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
index 65c2e13..82a86b4 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -178,60 +178,62 @@ public abstract class MultiTableInputFormatBase extends
List<InputSplit> splits = new ArrayList<>();
Iterator iter = tableMaps.entrySet().iterator();
- while (iter.hasNext()) {
- Map.Entry<TableName, List<Scan>> entry = (Map.Entry<TableName, List<Scan>>) iter.next();
- TableName tableName = entry.getKey();
- List<Scan> scanList = entry.getValue();
-
- try (Connection conn = ConnectionFactory.createConnection(context.getConfiguration());
- Table table = conn.getTable(tableName);
- RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
- RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
- regionLocator, conn.getAdmin());
- Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys();
- for (Scan scan : scanList) {
- if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
- throw new IOException("Expecting at least one region for table : "
- + tableName.getNameAsString());
- }
- int count = 0;
-
- byte[] startRow = scan.getStartRow();
- byte[] stopRow = scan.getStopRow();
-
- for (int i = 0; i < keys.getFirst().length; i++) {
- if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
- continue;
+ // Make a single Connection to the Cluster and use it across all tables.
+ try (Connection conn = ConnectionFactory.createConnection(context.getConfiguration())) {
+ while (iter.hasNext()) {
+ Map.Entry<TableName, List<Scan>> entry = (Map.Entry<TableName, List<Scan>>) iter.next();
+ TableName tableName = entry.getKey();
+ List<Scan> scanList = entry.getValue();
+ try (Table table = conn.getTable(tableName);
+ RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
+ RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
+ regionLocator, conn.getAdmin());
+ Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys();
+ for (Scan scan : scanList) {
+ if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
+ throw new IOException("Expecting at least one region for table : "
+ + tableName.getNameAsString());
}
-
- if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
- Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
- (stopRow.length == 0 || Bytes.compareTo(stopRow,
- keys.getFirst()[i]) > 0)) {
- byte[] splitStart = startRow.length == 0 ||
- Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
- keys.getFirst()[i] : startRow;
- byte[] splitStop = (stopRow.length == 0 ||
- Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
- keys.getSecond()[i].length > 0 ?
- keys.getSecond()[i] : stopRow;
-
- HRegionLocation hregionLocation = regionLocator.getRegionLocation(
- keys.getFirst()[i], false);
- String regionHostname = hregionLocation.getHostname();
- HRegionInfo regionInfo = hregionLocation.getRegionInfo();
- String encodedRegionName = regionInfo.getEncodedName();
- long regionSize = sizeCalculator.getRegionSize(
- regionInfo.getRegionName());
-
- TableSplit split = new TableSplit(table.getName(),
- scan, splitStart, splitStop, regionHostname,
- encodedRegionName, regionSize);
-
- splits.add(split);
-
- if (LOG.isDebugEnabled())
- LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+ int count = 0;
+
+ byte[] startRow = scan.getStartRow();
+ byte[] stopRow = scan.getStopRow();
+
+ for (int i = 0; i < keys.getFirst().length; i++) {
+ if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+ continue;
+ }
+
+ if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+ Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+ (stopRow.length == 0 || Bytes.compareTo(stopRow,
+ keys.getFirst()[i]) > 0)) {
+ byte[] splitStart = startRow.length == 0 ||
+ Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
+ keys.getFirst()[i] : startRow;
+ byte[] splitStop = (stopRow.length == 0 ||
+ Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
+ keys.getSecond()[i].length > 0 ?
+ keys.getSecond()[i] : stopRow;
+
+ HRegionLocation hregionLocation = regionLocator.getRegionLocation(
+ keys.getFirst()[i], false);
+ String regionHostname = hregionLocation.getHostname();
+ HRegionInfo regionInfo = hregionLocation.getRegionInfo();
+ String encodedRegionName = regionInfo.getEncodedName();
+ long regionSize = sizeCalculator.getRegionSize(
+ regionInfo.getRegionName());
+
+ TableSplit split = new TableSplit(table.getName(),
+ scan, splitStart, splitStop, regionHostname,
+ encodedRegionName, regionSize);
+
+ splits.add(split);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+ }
+ }
}
}
}