You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2014/12/18 19:55:10 UTC
hbase git commit: HBASE-12223 Revert from 1.0 branch
Repository: hbase
Updated Branches:
refs/heads/branch-1.0 9e18d8aa5 -> c86607fd7
HBASE-12223 Revert from 1.0 branch
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c86607fd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c86607fd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c86607fd
Branch: refs/heads/branch-1.0
Commit: c86607fd71a86904592a0c1789872fd40e2dafdb
Parents: 9e18d8a
Author: tedyu <yu...@gmail.com>
Authored: Thu Dec 18 10:55:02 2014 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Dec 18 10:55:02 2014 -0800
----------------------------------------------------------------------
.../mapreduce/MultiTableInputFormatBase.java | 101 ++++++++-----------
1 file changed, 41 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c86607fd/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
index b9a2db7..5c253cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -46,9 +46,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Iterator;
/**
* A base for {@link MultiTableInputFormat}s. Receives a list of
* {@link Scan} instances that define the input tables and
@@ -132,82 +129,67 @@ public abstract class MultiTableInputFormatBase extends
if (scans.isEmpty()) {
throw new IOException("No scans were provided.");
}
+ List<InputSplit> splits = new ArrayList<InputSplit>();
- Map<TableName, List<Scan>> tableMaps = new HashMap<TableName, List<Scan>>();
for (Scan scan : scans) {
byte[] tableNameBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
if (tableNameBytes == null)
throw new IOException("A scan object did not have a table name");
TableName tableName = TableName.valueOf(tableNameBytes);
-
- List<Scan> scanList = tableMaps.get(tableName);
- if (scanList == null) {
- scanList = new ArrayList<Scan>();
- tableMaps.put(tableName, scanList);
- }
- scanList.add(scan);
- }
-
- List<InputSplit> splits = new ArrayList<InputSplit>();
- Iterator iter = tableMaps.entrySet().iterator();
- while (iter.hasNext()) {
- Map.Entry<TableName, List<Scan>> entry = (Map.Entry<TableName, List<Scan>>) iter.next();
- TableName tableName = entry.getKey();
- List<Scan> scanList = entry.getValue();
Table table = null;
RegionLocator regionLocator = null;
Connection conn = null;
-
- try{
+ try {
conn = ConnectionFactory.createConnection(context.getConfiguration());
table = conn.getTable(tableName);
+ regionLocator = conn.getRegionLocator(tableName);
regionLocator = (RegionLocator) table;
- RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
- regionLocator, conn.getAdmin());
Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys();
- for (Scan scan : scanList) {
- if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
- throw new IOException("Expecting at least one region for table : "
- + tableName.getNameAsString());
- }
- int count = 0;
-
- byte[] startRow = scan.getStartRow();
- byte[] stopRow = scan.getStopRow();
+ if (keys == null || keys.getFirst() == null ||
+ keys.getFirst().length == 0) {
+ throw new IOException("Expecting at least one region for table : "
+ + tableName.getNameAsString());
+ }
+ int count = 0;
- for (int i = 0; i < keys.getFirst().length; i++) {
- if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
- continue;
- }
+ byte[] startRow = scan.getStartRow();
+ byte[] stopRow = scan.getStopRow();
- if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
- Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
- (stopRow.length == 0 || Bytes.compareTo(stopRow,
- keys.getFirst()[i]) > 0)) {
- byte[] splitStart = startRow.length == 0 ||
- Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
- keys.getFirst()[i] : startRow;
- byte[] splitStop = (stopRow.length == 0 ||
- Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
- keys.getSecond()[i].length > 0 ?
- keys.getSecond()[i] : stopRow;
+ RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
+ regionLocator, conn.getAdmin());
- HRegionLocation hregionLocation = regionLocator.getRegionLocation(
- keys.getFirst()[i], false);
- String regionHostname = hregionLocation.getHostname();
- HRegionInfo regionInfo = hregionLocation.getRegionInfo();
- long regionSize = sizeCalculator.getRegionSize(
- regionInfo.getRegionName());
+ for (int i = 0; i < keys.getFirst().length; i++) {
+ if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+ continue;
+ }
+ HRegionLocation hregionLocation = regionLocator.getRegionLocation(
+ keys.getFirst()[i], false);
+ String regionHostname = hregionLocation.getHostname();
+ HRegionInfo regionInfo = hregionLocation.getRegionInfo();
- TableSplit split = new TableSplit(table.getName(),
- scan, splitStart, splitStop, regionHostname, regionSize);
+ // determine if the given start and stop keys fall into the range
+ if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+ Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+ (stopRow.length == 0 ||
+ Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
+ byte[] splitStart =
+ startRow.length == 0 ||
+ Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
+ .getFirst()[i] : startRow;
+ byte[] splitStop =
+ (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
+ stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
+ .getSecond()[i] : stopRow;
- splits.add(split);
+ long regionSize = sizeCalculator.getRegionSize(regionInfo.getRegionName());
+ TableSplit split =
+ new TableSplit(regionLocator.getName(),
+ scan, splitStart, splitStop, regionHostname, regionSize);
- if (LOG.isDebugEnabled())
- LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
- }
+ splits.add(split);
+ if (LOG.isDebugEnabled())
+ LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
}
}
} finally {
@@ -216,7 +198,6 @@ public abstract class MultiTableInputFormatBase extends
if (null != conn) conn.close();
}
}
-
return splits;
}