You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/03/09 02:58:44 UTC
svn commit: r1575645 [1/2] - in /hbase/branches/0.96:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-common/src/test/java/org/apache/hadoop/hbase/
hbase-it/src/test/java/org/apache/h...
Author: stack
Date: Sun Mar 9 01:58:43 2014
New Revision: 1575645
URL: http://svn.apache.org/r1575645
Log:
HBASE-10707 Backport parent issue to 0.96
Added:
hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java
hbase/branches/0.96/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableSnapshotInputFormat.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/ScanPerformanceEvaluation.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
Modified:
hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java
hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
hbase/branches/0.96/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
hbase/branches/0.96/hbase-protocol/src/main/protobuf/MapReduce.proto
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
Modified: hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java (original)
+++ hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java Sun Mar 9 01:58:43 2014
@@ -18,9 +18,12 @@
package org.apache.hadoop.hbase.client;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* Helper class for custom client scanners.
@@ -28,6 +31,49 @@ import org.apache.hadoop.classification.
@InterfaceAudience.Private
public abstract class AbstractClientScanner implements ResultScanner {
+ protected ScanMetrics scanMetrics;
+
+ /**
+ * Check and initialize if application wants to collect scan metrics
+ */
+ protected void initScanMetrics(Scan scan) {
+ // check if application wants to collect scan metrics
+ byte[] enableMetrics = scan.getAttribute(
+ Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
+ if (enableMetrics != null && Bytes.toBoolean(enableMetrics)) {
+ scanMetrics = new ScanMetrics();
+ }
+ }
+
+ // TODO: should this be at ResultScanner? ScanMetrics is not public API it seems.
+ public ScanMetrics getScanMetrics() {
+ return scanMetrics;
+ }
+
+ /**
+ * Get <param>nbRows</param> rows.
+ * How many RPCs are made is determined by the {@link Scan#setCaching(int)}
+ * setting (or hbase.client.scanner.caching in hbase-site.xml).
+ * @param nbRows number of rows to return
+ * @return Between zero and <param>nbRows</param> RowResults. Scan is done
+ * if returned array is of zero-length (We never return null).
+ * @throws IOException
+ */
+ @Override
+ public Result [] next(int nbRows) throws IOException {
+ // Collect values to be returned here
+ ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
+ for(int i = 0; i < nbRows; i++) {
+ Result next = next();
+ if (next != null) {
+ resultSets.add(next);
+ } else {
+ break;
+ }
+ }
+ return resultSets.toArray(new Result[resultSets.size()]);
+ }
+
@Override
public Iterator<Result> iterator() {
return new Iterator<Result>() {
@@ -38,6 +84,7 @@ public abstract class AbstractClientScan
// this method is where the actual advancing takes place, but you need
// to call next() to consume it. hasNext() will only advance if there
// isn't a pending next().
+ @Override
public boolean hasNext() {
if (next == null) {
try {
@@ -52,6 +99,7 @@ public abstract class AbstractClientScan
// get the pending next item and advance the iterator. returns null if
// there is no next item.
+ @Override
public Result next() {
// since hasNext() does the real advancing, we call this to determine
// if there is a next before proceeding.
@@ -67,6 +115,7 @@ public abstract class AbstractClientScan
return temp;
}
+ @Override
public void remove() {
throw new UnsupportedOperationException();
}
Modified: hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java (original)
+++ hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java Sun Mar 9 01:58:43 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.client;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.LinkedList;
import org.apache.commons.logging.Log;
@@ -35,7 +34,6 @@ import org.apache.hadoop.hbase.KeyValueU
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownScannerException;
-import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
@@ -62,7 +60,6 @@ public class ClientScanner extends Abstr
protected long lastNext;
// Keep lastResult returned successfully in case we have to reset scanner.
protected Result lastResult = null;
- protected ScanMetrics scanMetrics = null;
protected final long maxScannerResultSize;
private final HConnection connection;
private final TableName tableName;
@@ -151,11 +148,7 @@ public class ClientScanner extends Abstr
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
// check if application wants to collect scan metrics
- byte[] enableMetrics = scan.getAttribute(
- Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
- if (enableMetrics != null && Bytes.toBoolean(enableMetrics)) {
- scanMetrics = new ScanMetrics();
- }
+ initScanMetrics(scan);
// Use the caching from the Scan. If not set, use the default cache setting for this table.
if (this.scan.getCaching() > 0) {
@@ -170,7 +163,7 @@ public class ClientScanner extends Abstr
initializeScannerInConstruction();
}
-
+
protected void initializeScannerInConstruction() throws IOException{
// initialize the scanner
nextScanner(this.caching, false);
@@ -429,30 +422,6 @@ public class ClientScanner extends Abstr
return null;
}
- /**
- * Get <param>nbRows</param> rows.
- * How many RPCs are made is determined by the {@link Scan#setCaching(int)}
- * setting (or hbase.client.scanner.caching in hbase-site.xml).
- * @param nbRows number of rows to return
- * @return Between zero and <param>nbRows</param> RowResults. Scan is done
- * if returned array is of zero-length (We never return null).
- * @throws IOException
- */
- @Override
- public Result [] next(int nbRows) throws IOException {
- // Collect values to be returned here
- ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
- for(int i = 0; i < nbRows; i++) {
- Result next = next();
- if (next != null) {
- resultSets.add(next);
- } else {
- break;
- }
- }
- return resultSets.toArray(new Result[resultSets.size()]);
- }
-
@Override
public void close() {
if (!scanMetricsPublished) writeScanMetrics();
Modified: hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java (original)
+++ hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java Sun Mar 9 01:58:43 2014
@@ -333,4 +333,16 @@ public final class CellUtil {
// Serialization is probably preceded by a length (it is in the KeyValueCodec at least).
Bytes.SIZEOF_INT;
}
+
+ /**
+ * Returns true if the first range start1...end1 overlaps with the second range
+ * start2...end2, assuming the byte arrays represent row keys
+ */
+ public static boolean overlappingKeys(final byte[] start1, final byte[] end1,
+ final byte[] start2, final byte[] end2) {
+ return (end2.length == 0 || start1.length == 0 || Bytes.compareTo(start1,
+ end2) < 0)
+ && (end1.length == 0 || start2.length == 0 || Bytes.compareTo(start2,
+ end1) < 0);
+ }
}
Added: hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java?rev=1575645&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java (added)
+++ hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java Sun Mar 9 01:58:43 2014
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestCellUtil {
+
+ @Test
+ public void testOverlappingKeys() {
+ byte[] empty = HConstants.EMPTY_BYTE_ARRAY;
+ byte[] a = Bytes.toBytes("a");
+ byte[] b = Bytes.toBytes("b");
+ byte[] c = Bytes.toBytes("c");
+ byte[] d = Bytes.toBytes("d");
+
+ // overlaps
+ Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, b));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, c, a, b));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, c));
+ Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, c));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, c, b, c));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, d, b, c));
+ Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, d));
+
+ Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, b));
+ Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, c));
+
+ Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, b));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, c));
+
+ Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, b));
+ Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, c));
+
+ Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, empty));
+ Assert.assertTrue(CellUtil.overlappingKeys(empty, empty, a, b));
+
+ // non overlaps
+ Assert.assertFalse(CellUtil.overlappingKeys(a, b, c, d));
+ Assert.assertFalse(CellUtil.overlappingKeys(c, d, a, b));
+
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, d));
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, empty));
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, d, empty));
+ Assert.assertFalse(CellUtil.overlappingKeys(c, d, b, c));
+ Assert.assertFalse(CellUtil.overlappingKeys(c, empty, b, c));
+ Assert.assertFalse(CellUtil.overlappingKeys(d, empty, b, c));
+
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, a, b));
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, b));
+ Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, a));
+ Assert.assertFalse(CellUtil.overlappingKeys(a,b, b, c));
+ Assert.assertFalse(CellUtil.overlappingKeys(empty, b, b, c));
+ Assert.assertFalse(CellUtil.overlappingKeys(empty, a, b, c));
+ }
+}
Added: hbase/branches/0.96/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableSnapshotInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableSnapshotInputFormat.java?rev=1575645&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableSnapshotInputFormat.java (added)
+++ hbase/branches/0.96/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableSnapshotInputFormat.java Sun Mar 9 01:58:43 2014
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.IntegrationTestBase;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.IntegrationTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.experimental.categories.Category;
+
+/**
+ * An integration test to test {@link TableSnapshotInputFormat} which enables
+ * reading directly from snapshot files without going through hbase servers.
+ *
+ * This test creates a table and loads the table with the rows ranging from
+ * 'aaa' to 'zzz', and for each row, sets the columns f1:(null) and f2:(null) to be
+ * the the same as the row value.
+ * <pre>
+ * aaa, f1: => aaa
+ * aaa, f2: => aaa
+ * aab, f1: => aab
+ * ....
+ * zzz, f2: => zzz
+ * </pre>
+ *
+ * Then the test creates a snapshot from this table, and overrides the values in the original
+ * table with values 'after_snapshot_value'. The test, then runs a mapreduce job over the snapshot
+ * with a scan start row 'bbb' and stop row 'yyy'. The data is saved in a single reduce output file, and
+ * inspected later to verify that the MR job has seen all the values from the snapshot.
+ *
+ * <p> These parameters can be used to configure the job:
+ * <br>"IntegrationTestTableSnapshotInputFormat.table" => the name of the table
+ * <br>"IntegrationTestTableSnapshotInputFormat.snapshot" => the name of the snapshot
+ * <br>"IntegrationTestTableSnapshotInputFormat.numRegions" => number of regions in the table to be created
+ * <br>"IntegrationTestTableSnapshotInputFormat.tableDir" => temporary directory to restore the snapshot files
+ *
+ */
+@Category(IntegrationTests.class)
+// Not runnable as a unit test. See TestTableSnapshotInputFormat
+public class IntegrationTestTableSnapshotInputFormat extends IntegrationTestBase {
+
+ private static final Log LOG = LogFactory.getLog(IntegrationTestTableSnapshotInputFormat.class);
+
+ private static final String TABLE_NAME_KEY = "IntegrationTestTableSnapshotInputFormat.table";
+ private static final String DEFAULT_TABLE_NAME = "IntegrationTestTableSnapshotInputFormat";
+
+ private static final String SNAPSHOT_NAME_KEY = "IntegrationTestTableSnapshotInputFormat.snapshot";
+
+
+ private static final String NUM_REGIONS_KEY = "IntegrationTestTableSnapshotInputFormat.numRegions";
+ private static final int DEFAULT_NUM_REGIONS = 32;
+
+ private static final String TABLE_DIR_KEY = "IntegrationTestTableSnapshotInputFormat.tableDir";
+
+ private IntegrationTestingUtility util;
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ util = getTestingUtil(conf);
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ util = getTestingUtil(getConf());
+ util.initializeCluster(1);
+ this.setConf(util.getConfiguration());
+ }
+
+ @Override
+ @After
+ public void cleanUp() throws Exception {
+ util.restoreCluster();
+ }
+
+ @Override
+ public void setUpCluster() throws Exception {
+ }
+
+ @Override
+ public int runTestFromCommandLine() throws Exception {
+ Configuration conf = getConf();
+ TableName tableName = TableName.valueOf(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME));
+ String snapshotName = conf.get(SNAPSHOT_NAME_KEY, tableName.getQualifierAsString()
+ + "_snapshot_" + System.currentTimeMillis());
+ int numRegions = conf.getInt(NUM_REGIONS_KEY, DEFAULT_NUM_REGIONS);
+ String tableDirStr = conf.get(TABLE_DIR_KEY);
+ Path tableDir;
+ if (tableDirStr == null) {
+ tableDir = util.getDataTestDirOnTestFS(tableName.getQualifierAsString());
+ } else {
+ tableDir = new Path(tableDirStr);
+ }
+
+ /* We create the table using HBaseAdmin#createTable(), which will create the table
+ * with desired number of regions. We pass bbb as startKey and yyy as endKey, so if
+ * desiredNumRegions is > 2, we create regions empty - bbb and yyy - empty, and we
+ * create numRegions - 2 regions between bbb - yyy. The test uses a Scan with startRow
+ * bbb and endRow yyy, so, we expect the first and last region to be filtered out in
+ * the input format, and we expect numRegions - 2 splits between bbb and yyy.
+ */
+ int expectedNumSplits = numRegions > 2 ? numRegions - 2 : numRegions;
+
+ TestTableSnapshotInputFormat.doTestWithMapReduce(util, tableName, snapshotName, tableDir,
+ numRegions, expectedNumSplits, false);
+
+ return 0;
+ }
+
+ @Override // CM is not intended to be run with this test
+ public String getTablename() {
+ return null;
+ }
+
+ @Override
+ protected Set<String> getColumnFamilies() {
+ return null;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ IntegrationTestingUtility.setUseDistributedCluster(conf);
+ int ret = ToolRunner.run(conf, new IntegrationTestTableSnapshotInputFormat(), args);
+ System.exit(ret);
+ }
+
+}
Modified: hbase/branches/0.96/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java (original)
+++ hbase/branches/0.96/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java Sun Mar 9 01:58:43 2014
@@ -717,11 +717,764 @@ public final class MapReduceProtos {
// @@protoc_insertion_point(class_scope:ScanMetrics)
}
+ public interface TableSnapshotRegionSplitOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional .RegionSpecifier region = 1;
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ boolean hasRegion();
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion();
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder();
+
+ // repeated string locations = 2;
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ java.util.List<java.lang.String>
+ getLocationsList();
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ int getLocationsCount();
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ java.lang.String getLocations(int index);
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ com.google.protobuf.ByteString
+ getLocationsBytes(int index);
+ }
+ /**
+ * Protobuf type {@code TableSnapshotRegionSplit}
+ */
+ public static final class TableSnapshotRegionSplit extends
+ com.google.protobuf.GeneratedMessage
+ implements TableSnapshotRegionSplitOrBuilder {
+ // Use TableSnapshotRegionSplit.newBuilder() to construct.
+ private TableSnapshotRegionSplit(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private TableSnapshotRegionSplit(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final TableSnapshotRegionSplit defaultInstance;
+ public static TableSnapshotRegionSplit getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public TableSnapshotRegionSplit getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private TableSnapshotRegionSplit(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ subBuilder = region_.toBuilder();
+ }
+ region_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(region_);
+ region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000001;
+ break;
+ }
+ case 18: {
+ if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
+ locations_ = new com.google.protobuf.LazyStringArrayList();
+ mutable_bitField0_ |= 0x00000002;
+ }
+ locations_.add(input.readBytes());
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
+ locations_ = new com.google.protobuf.UnmodifiableLazyStringList(locations_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_TableSnapshotRegionSplit_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_TableSnapshotRegionSplit_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.class, org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<TableSnapshotRegionSplit> PARSER =
+ new com.google.protobuf.AbstractParser<TableSnapshotRegionSplit>() {
+ public TableSnapshotRegionSplit parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new TableSnapshotRegionSplit(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<TableSnapshotRegionSplit> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional .RegionSpecifier region = 1;
+ public static final int REGION_FIELD_NUMBER = 1;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier region_;
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public boolean hasRegion() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion() {
+ return region_;
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder() {
+ return region_;
+ }
+
+ // repeated string locations = 2;
+ public static final int LOCATIONS_FIELD_NUMBER = 2;
+ private com.google.protobuf.LazyStringList locations_;
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public java.util.List<java.lang.String>
+ getLocationsList() {
+ return locations_;
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public int getLocationsCount() {
+ return locations_.size();
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public java.lang.String getLocations(int index) {
+ return locations_.get(index);
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getLocationsBytes(int index) {
+ return locations_.getByteString(index);
+ }
+
+ private void initFields() {
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
+ locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (hasRegion()) {
+ if (!getRegion().isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeMessage(1, region_);
+ }
+ for (int i = 0; i < locations_.size(); i++) {
+ output.writeBytes(2, locations_.getByteString(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(1, region_);
+ }
+ {
+ int dataSize = 0;
+ for (int i = 0; i < locations_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeBytesSizeNoTag(locations_.getByteString(i));
+ }
+ size += dataSize;
+ size += 1 * getLocationsList().size();
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit other = (org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit) obj;
+
+ boolean result = true;
+ result = result && (hasRegion() == other.hasRegion());
+ if (hasRegion()) {
+ result = result && getRegion()
+ .equals(other.getRegion());
+ }
+ result = result && getLocationsList()
+ .equals(other.getLocationsList());
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasRegion()) {
+ hash = (37 * hash) + REGION_FIELD_NUMBER;
+ hash = (53 * hash) + getRegion().hashCode();
+ }
+ if (getLocationsCount() > 0) {
+ hash = (37 * hash) + LOCATIONS_FIELD_NUMBER;
+ hash = (53 * hash) + getLocationsList().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code TableSnapshotRegionSplit}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplitOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_TableSnapshotRegionSplit_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_TableSnapshotRegionSplit_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.class, org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getRegionFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ if (regionBuilder_ == null) {
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
+ } else {
+ regionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000001);
+ locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_TableSnapshotRegionSplit_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit build() {
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit result = new org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ if (regionBuilder_ == null) {
+ result.region_ = region_;
+ } else {
+ result.region_ = regionBuilder_.build();
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ locations_ = new com.google.protobuf.UnmodifiableLazyStringList(
+ locations_);
+ bitField0_ = (bitField0_ & ~0x00000002);
+ }
+ result.locations_ = locations_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.getDefaultInstance()) return this;
+ if (other.hasRegion()) {
+ mergeRegion(other.getRegion());
+ }
+ if (!other.locations_.isEmpty()) {
+ if (locations_.isEmpty()) {
+ locations_ = other.locations_;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ } else {
+ ensureLocationsIsMutable();
+ locations_.addAll(other.locations_);
+ }
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (hasRegion()) {
+ if (!getRegion().isInitialized()) {
+
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional .RegionSpecifier region = 1;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder> regionBuilder_;
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public boolean hasRegion() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion() {
+ if (regionBuilder_ == null) {
+ return region_;
+ } else {
+ return regionBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public Builder setRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier value) {
+ if (regionBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ region_ = value;
+ onChanged();
+ } else {
+ regionBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000001;
+ return this;
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public Builder setRegion(
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder builderForValue) {
+ if (regionBuilder_ == null) {
+ region_ = builderForValue.build();
+ onChanged();
+ } else {
+ regionBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000001;
+ return this;
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public Builder mergeRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier value) {
+ if (regionBuilder_ == null) {
+ if (((bitField0_ & 0x00000001) == 0x00000001) &&
+ region_ != org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance()) {
+ region_ =
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.newBuilder(region_).mergeFrom(value).buildPartial();
+ } else {
+ region_ = value;
+ }
+ onChanged();
+ } else {
+ regionBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000001;
+ return this;
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public Builder clearRegion() {
+ if (regionBuilder_ == null) {
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
+ onChanged();
+ } else {
+ regionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder getRegionBuilder() {
+ bitField0_ |= 0x00000001;
+ onChanged();
+ return getRegionFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder() {
+ if (regionBuilder_ != null) {
+ return regionBuilder_.getMessageOrBuilder();
+ } else {
+ return region_;
+ }
+ }
+ /**
+ * <code>optional .RegionSpecifier region = 1;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder>
+ getRegionFieldBuilder() {
+ if (regionBuilder_ == null) {
+ regionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder>(
+ region_,
+ getParentForChildren(),
+ isClean());
+ region_ = null;
+ }
+ return regionBuilder_;
+ }
+
+ // repeated string locations = 2;
+ private com.google.protobuf.LazyStringList locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ private void ensureLocationsIsMutable() {
+ if (!((bitField0_ & 0x00000002) == 0x00000002)) {
+ locations_ = new com.google.protobuf.LazyStringArrayList(locations_);
+ bitField0_ |= 0x00000002;
+ }
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public java.util.List<java.lang.String>
+ getLocationsList() {
+ return java.util.Collections.unmodifiableList(locations_);
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public int getLocationsCount() {
+ return locations_.size();
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public java.lang.String getLocations(int index) {
+ return locations_.get(index);
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getLocationsBytes(int index) {
+ return locations_.getByteString(index);
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public Builder setLocations(
+ int index, java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureLocationsIsMutable();
+ locations_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public Builder addLocations(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureLocationsIsMutable();
+ locations_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public Builder addAllLocations(
+ java.lang.Iterable<java.lang.String> values) {
+ ensureLocationsIsMutable();
+ super.addAll(values, locations_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public Builder clearLocations() {
+ locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated string locations = 2;</code>
+ */
+ public Builder addLocationsBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureLocationsIsMutable();
+ locations_.add(value);
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:TableSnapshotRegionSplit)
+ }
+
+ static {
+ defaultInstance = new TableSnapshotRegionSplit(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:TableSnapshotRegionSplit)
+ }
+
private static com.google.protobuf.Descriptors.Descriptor
internal_static_ScanMetrics_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
internal_static_ScanMetrics_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_TableSnapshotRegionSplit_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_TableSnapshotRegionSplit_fieldAccessorTable;
public static com.google.protobuf.Descriptors.FileDescriptor
getDescriptor() {
@@ -732,9 +1485,11 @@ public final class MapReduceProtos {
static {
java.lang.String[] descriptorData = {
"\n\017MapReduce.proto\032\013HBase.proto\".\n\013ScanMe" +
- "trics\022\037\n\007metrics\030\001 \003(\0132\016.NameInt64PairBB" +
- "\n*org.apache.hadoop.hbase.protobuf.gener" +
- "atedB\017MapReduceProtosH\001\240\001\001"
+ "trics\022\037\n\007metrics\030\001 \003(\0132\016.NameInt64Pair\"O" +
+ "\n\030TableSnapshotRegionSplit\022 \n\006region\030\001 \001" +
+ "(\0132\020.RegionSpecifier\022\021\n\tlocations\030\002 \003(\tB" +
+ "B\n*org.apache.hadoop.hbase.protobuf.gene" +
+ "ratedB\017MapReduceProtosH\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -747,6 +1502,12 @@ public final class MapReduceProtos {
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_ScanMetrics_descriptor,
new java.lang.String[] { "Metrics", });
+ internal_static_TableSnapshotRegionSplit_descriptor =
+ getDescriptor().getMessageTypes().get(1);
+ internal_static_TableSnapshotRegionSplit_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_TableSnapshotRegionSplit_descriptor,
+ new java.lang.String[] { "Region", "Locations", });
return null;
}
};
Modified: hbase/branches/0.96/hbase-protocol/src/main/protobuf/MapReduce.proto
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-protocol/src/main/protobuf/MapReduce.proto?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-protocol/src/main/protobuf/MapReduce.proto (original)
+++ hbase/branches/0.96/hbase-protocol/src/main/protobuf/MapReduce.proto Sun Mar 9 01:58:43 2014
@@ -18,15 +18,18 @@
//This file includes protocol buffers used in MapReduce only.
- option java_package = "org.apache.hadoop.hbase.protobuf.generated";
- option java_outer_classname = "MapReduceProtos";
- option java_generate_equals_and_hash = true;
- option optimize_for = SPEED;
-
- import "HBase.proto";
-
- message ScanMetrics {
-
- repeated NameInt64Pair metrics = 1;
-
- }
\ No newline at end of file
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MapReduceProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message ScanMetrics {
+ repeated NameInt64Pair metrics = 1;
+}
+
+message TableSnapshotRegionSplit {
+ optional RegionSpecifier region = 1;
+ repeated string locations = 2;
+}
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java Sun Mar 9 01:58:43 2014
@@ -38,7 +38,7 @@ import org.apache.hadoop.classification.
public class HDFSBlocksDistribution {
private Map<String,HostAndWeight> hostAndWeights = null;
private long uniqueBlocksTotalWeight = 0;
-
+
/**
* Stores the hostname and weight for that hostname.
*
@@ -58,7 +58,7 @@ public class HDFSBlocksDistribution {
* Constructor
* @param host the host name
* @param weight the weight
- */
+ */
public HostAndWeight(String host, long weight) {
this.host = host;
this.weight = weight;
@@ -67,28 +67,28 @@ public class HDFSBlocksDistribution {
/**
* add weight
* @param weight the weight
- */
+ */
public void addWeight(long weight) {
this.weight += weight;
}
/**
* @return the host name
- */
+ */
public String getHost() {
return host;
}
/**
* @return the weight
- */
+ */
public long getWeight() {
return weight;
}
/**
* comparator used to sort hosts based on weight
- */
+ */
public static class WeightComparator implements Comparator<HostAndWeight> {
@Override
public int compare(HostAndWeight l, HostAndWeight r) {
@@ -99,7 +99,7 @@ public class HDFSBlocksDistribution {
}
}
}
-
+
/**
* Constructor
*/
@@ -137,12 +137,12 @@ public class HDFSBlocksDistribution {
/**
* add some weight to the total unique weight
* @param weight the weight
- */
+ */
private void addUniqueWeight(long weight) {
uniqueBlocksTotalWeight += weight;
}
-
-
+
+
/**
* add some weight to a specific host
* @param host the host name
@@ -186,14 +186,14 @@ public class HDFSBlocksDistribution {
}
return weight;
}
-
+
/**
* @return the sum of all unique blocks' weight
*/
public long getUniqueBlocksTotalWeight() {
return uniqueBlocksTotalWeight;
}
-
+
/**
* return the locality index of a given host
* @param host the host name
@@ -207,8 +207,8 @@ public class HDFSBlocksDistribution {
}
return localityIndex;
}
-
-
+
+
/**
* This will add the distribution from input to this object
* @param otherBlocksDistribution the other hdfs blocks distribution
@@ -223,19 +223,27 @@ public class HDFSBlocksDistribution {
}
addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight());
}
-
+
/**
* return the sorted list of hosts in terms of their weights
*/
public List<String> getTopHosts() {
- NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
- new HostAndWeight.WeightComparator());
- orderedHosts.addAll(this.hostAndWeights.values());
- List<String> topHosts = new ArrayList<String>(orderedHosts.size());
- for(HostAndWeight haw : orderedHosts.descendingSet()) {
+ HostAndWeight[] hostAndWeights = getTopHostsWithWeights();
+ List<String> topHosts = new ArrayList<String>(hostAndWeights.length);
+ for(HostAndWeight haw : hostAndWeights) {
topHosts.add(haw.getHost());
}
return topHosts;
}
+ /**
+ * return the sorted list of hosts in terms of their weights
+ */
+ public HostAndWeight[] getTopHostsWithWeights() {
+ NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
+ new HostAndWeight.WeightComparator());
+ orderedHosts.addAll(this.hostAndWeights.values());
+ return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]);
+ }
+
}
Added: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java?rev=1575645&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java (added)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java Sun Mar 9 01:58:43 2014
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.mortbay.log.Log;
+
+/**
+ * A client scanner for a region opened for read-only on the client side. Assumes region data
+ * is not changing.
+ */
+@InterfaceAudience.Private
+public class ClientSideRegionScanner extends AbstractClientScanner {
+
+ private HRegion region;
+ private Scan scan;
+ RegionScanner scanner;
+ List<Cell> values;
+
+ public ClientSideRegionScanner(Configuration conf, FileSystem fs,
+ Path rootDir, HTableDescriptor htd, HRegionInfo hri, Scan scan, ScanMetrics scanMetrics) throws IOException {
+
+ this.scan = scan;
+
+ // region is immutable, set isolation level
+ scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
+
+ // open region from the snapshot directory
+ this.region = HRegion.openHRegion(conf, fs, rootDir, hri, htd, null, null, null);
+
+ // create an internal region scanner
+ this.scanner = region.getScanner(scan);
+ values = new ArrayList<Cell>();
+
+ if (scanMetrics == null) {
+ initScanMetrics(scan);
+ } else {
+ this.scanMetrics = scanMetrics;
+ }
+ region.startRegionOperation();
+ }
+
+ @Override
+ public Result next() throws IOException {
+ values.clear();
+
+ scanner.nextRaw(values, -1); // pass -1 as limit so that we see the whole row.
+ if (values == null || values.isEmpty()) {
+ //we are done
+ return null;
+ }
+
+ Result result = Result.create(values);
+ if (this.scanMetrics != null) {
+ long resultSize = 0;
+ for (Cell kv : values) {
+ // TODO add getLength to Cell/use CellUtil#estimatedSizeOf
+ resultSize += KeyValueUtil.ensureKeyValue(kv).getLength();
+ }
+ this.scanMetrics.countOfBytesInResults.addAndGet(resultSize);
+ }
+
+ return result;
+ }
+
+ @Override
+ public void close() {
+ if (this.scanner != null) {
+ try {
+ this.scanner.close();
+ this.scanner = null;
+ } catch (IOException ex) {
+ Log.warn("Exception while closing scanner", ex);
+ }
+ }
+ if (this.region != null) {
+ this.region.closeRegionOperation();
+ try {
+ this.region.close(true);
+ this.region = null;
+ } catch (IOException ex) {
+ Log.warn("Exception while closing region", ex);
+ }
+ }
+ }
+}
Added: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java?rev=1575645&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java (added)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java Sun Mar 9 01:58:43 2014
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+
+/**
+ * A Scanner which performs a scan over snapshot files. Using this class requires copying the
+ * snapshot to a temporary empty directory, which will copy the snapshot reference files into that
+ * directory. Actual data files are not copied.
+ *
+ * <p>
+ * This also allows one to run the scan from an
+ * online or offline hbase cluster. The snapshot files can be exported by using the
+ * {@link ExportSnapshot} tool, to a pure-hdfs cluster, and this scanner can be used to
+ * run the scan directly over the snapshot files. The snapshot should not be deleted while there
+ * are open scanners reading from snapshot files.
+ *
+ * <p>
+ * An internal RegionScanner is used to execute the {@link Scan} obtained
+ * from the user for each region in the snapshot.
+ * <p>
+ * HBase owns all the data and snapshot files on the filesystem. Only the HBase user can read from
+ * snapshot files and data files. HBase also enforces security because all the requests are handled
+ * by the server layer, and the user cannot read from the data files directly. To read from snapshot
+ * files directly from the file system, the user who is running the MR job must have sufficient
+ * permissions to access snapshot and reference files. This means that to run mapreduce over
+ * snapshot files, the job has to be run as the HBase user or the user must have group or other
+ * priviledges in the filesystem (See HBASE-8369). Note that, given other users access to read from
+ * snapshot/data files will completely circumvent the access control enforced by HBase.
+ * @see TableSnapshotInputFormat
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TableSnapshotScanner extends AbstractClientScanner {
+
+ private static final Log LOG = LogFactory.getLog(TableSnapshotScanner.class);
+
+ private Configuration conf;
+ private String snapshotName;
+ private FileSystem fs;
+ private Path rootDir;
+ private Path restoreDir;
+ private Scan scan;
+ private ArrayList<HRegionInfo> regions;
+ private HTableDescriptor htd;
+
+ private ClientSideRegionScanner currentRegionScanner = null;
+ private int currentRegion = -1;
+
+ /**
+ * Creates a TableSnapshotScanner.
+ * @param conf the configuration
+ * @param restoreDir a temporary directory to copy the snapshot files into. Current user should
+ * have write permissions to this directory, and this should not be a subdirectory of rootdir.
+ * The scanner deletes the contents of the directory once the scanner is closed.
+ * @param snapshotName the name of the snapshot to read from
+ * @param scan a Scan representing scan parameters
+ * @throws IOException in case of error
+ */
+ public TableSnapshotScanner(Configuration conf, Path restoreDir,
+ String snapshotName, Scan scan) throws IOException {
+ this(conf, new Path(conf.get(HConstants.HBASE_DIR)),
+ restoreDir, snapshotName, scan);
+ }
+
+ /**
+ * Creates a TableSnapshotScanner.
+ * @param conf the configuration
+ * @param rootDir root directory for HBase.
+ * @param restoreDir a temporary directory to copy the snapshot files into. Current user should
+ * have write permissions to this directory, and this should not be a subdirectory of rootdir.
+ * The scanner deletes the contents of the directory once the scanner is closed.
+ * @param snapshotName the name of the snapshot to read from
+ * @param scan a Scan representing scan parameters
+ * @throws IOException in case of error
+ */
+ public TableSnapshotScanner(Configuration conf, Path rootDir,
+ Path restoreDir, String snapshotName, Scan scan) throws IOException {
+ this.conf = conf;
+ this.snapshotName = snapshotName;
+ this.rootDir = rootDir;
+ // restoreDir will be deleted in close(), use a unique sub directory
+ this.restoreDir = new Path(restoreDir, UUID.randomUUID().toString());
+ this.scan = scan;
+ this.fs = rootDir.getFileSystem(conf);
+ init();
+ }
+
+ private void init() throws IOException {
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+
+ //load table descriptor
+ htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+
+ Set<String> snapshotRegionNames
+ = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
+ if (snapshotRegionNames == null) {
+ throw new IllegalArgumentException("Snapshot seems empty");
+ }
+
+ regions = new ArrayList<HRegionInfo>(snapshotRegionNames.size());
+ for (String regionName : snapshotRegionNames) {
+ // load region descriptor
+ Path regionDir = new Path(snapshotDir, regionName);
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
+ regionDir);
+
+ if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
+ hri.getStartKey(), hri.getEndKey())) {
+ regions.add(hri);
+ }
+ }
+
+ // sort for regions according to startKey.
+ Collections.sort(regions);
+
+ initScanMetrics(scan);
+
+ RestoreSnapshotHelper.copySnapshotForScanner(conf, fs,
+ rootDir, restoreDir, snapshotName);
+ }
+
+ @Override
+ public Result next() throws IOException {
+ Result result = null;
+ while (true) {
+ if (currentRegionScanner == null) {
+ currentRegion++;
+ if (currentRegion >= regions.size()) {
+ return null;
+ }
+
+ HRegionInfo hri = regions.get(currentRegion);
+ currentRegionScanner = new ClientSideRegionScanner(conf, fs,
+ restoreDir, htd, hri, scan, scanMetrics);
+ if (this.scanMetrics != null) {
+ this.scanMetrics.countOfRegions.incrementAndGet();
+ }
+ }
+
+ result = currentRegionScanner.next();
+ if (result != null) {
+ return result;
+ } else {
+ currentRegionScanner.close();
+ currentRegionScanner = null;
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ if (currentRegionScanner != null) {
+ currentRegionScanner.close();
+ }
+ try {
+ fs.delete(this.restoreDir, true);
+ } catch (IOException ex) {
+ LOG.warn("Could not delete restore directory for the snapshot:" + ex);
+ }
+ }
+
+}
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java Sun Mar 9 01:58:43 2014
@@ -66,6 +66,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.KeeperException;
+import org.cliffc.high_scale_lib.Counter;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -122,6 +123,32 @@ public class TableMapReduceUtil {
job, true);
}
+ /**
+ * Use this before submitting a TableMap job. It will appropriately set up
+ * the job.
+ *
+ * @param table The table name to read from.
+ * @param scan The scan instance with the columns, time range etc.
+ * @param mapper The mapper class to use.
+ * @param outputKeyClass The class of the output key.
+ * @param outputValueClass The class of the output value.
+ * @param job The current job to adjust. Make sure the passed job is
+ * carrying all necessary HBase configuration.
+ * @param addDependencyJars upload HBase jars and jars for any of the configured
+ * job classes via the distributed cache (tmpjars).
+ * @throws IOException When setting up the details fails.
+ */
+ public static void initTableMapperJob(String table, Scan scan,
+ Class<? extends TableMapper> mapper,
+ Class<?> outputKeyClass,
+ Class<?> outputValueClass, Job job,
+ boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
+ throws IOException {
+ initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass, job,
+ addDependencyJars, true, inputFormatClass);
+ }
+
+
/**
* Use this before submitting a TableMap job. It will appropriately set up
* the job.
@@ -135,13 +162,16 @@ public class TableMapReduceUtil {
* carrying all necessary HBase configuration.
* @param addDependencyJars upload HBase jars and jars for any of the configured
* job classes via the distributed cache (tmpjars).
+ * @param initCredentials whether to initialize hbase auth credentials for the job
+ * @param inputFormatClass the input format
* @throws IOException When setting up the details fails.
*/
public static void initTableMapperJob(String table, Scan scan,
Class<? extends TableMapper> mapper,
Class<?> outputKeyClass,
Class<?> outputValueClass, Job job,
- boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
+ boolean addDependencyJars, boolean initCredentials,
+ Class<? extends InputFormat> inputFormatClass)
throws IOException {
job.setInputFormatClass(inputFormatClass);
if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
@@ -160,7 +190,9 @@ public class TableMapReduceUtil {
if (addDependencyJars) {
addDependencyJars(job);
}
- initCredentials(job);
+ if (initCredentials) {
+ initCredentials(job);
+ }
}
/**
@@ -240,6 +272,40 @@ public class TableMapReduceUtil {
}
/**
+ * Sets up the job for reading from a table snapshot. It bypasses hbase servers
+ * and read directly from snapshot files.
+ *
+ * @param snapshotName The name of the snapshot (of a table) to read from.
+ * @param scan The scan instance with the columns, time range etc.
+ * @param mapper The mapper class to use.
+ * @param outputKeyClass The class of the output key.
+ * @param outputValueClass The class of the output value.
+ * @param job The current job to adjust. Make sure the passed job is
+ * carrying all necessary HBase configuration.
+ * @param addDependencyJars upload HBase jars and jars for any of the configured
+ * job classes via the distributed cache (tmpjars).
+ *
+ * @param tmpRestoreDir a temporary directory to copy the snapshot files into. Current user should
+ * have write permissions to this directory, and this should not be a subdirectory of rootdir.
+ * After the job is finished, restore directory can be deleted.
+ * @throws IOException When setting up the details fails.
+ * @see TableSnapshotInputFormat
+ */
+ public static void initTableSnapshotMapperJob(String snapshotName, Scan scan,
+ Class<? extends TableMapper> mapper,
+ Class<?> outputKeyClass,
+ Class<?> outputValueClass, Job job,
+ boolean addDependencyJars, Path tmpRestoreDir)
+ throws IOException {
+ TableSnapshotInputFormat.setInput(job, snapshotName, tmpRestoreDir);
+ initTableMapperJob(snapshotName, scan, mapper, outputKeyClass,
+ outputValueClass, job, addDependencyJars, false, TableSnapshotInputFormat.class);
+
+ // We would need even more libraries that hbase-server depends on
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Counter.class);
+ }
+
+ /**
* Use this before submitting a Multi TableMap job. It will appropriately set
* up the job.
*
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java?rev=1575645&r1=1575644&r2=1575645&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java Sun Mar 9 01:58:43 2014
@@ -35,11 +35,9 @@ import org.apache.hadoop.hbase.client.me
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
import org.apache.hadoop.util.StringUtils;
/**
@@ -103,7 +101,7 @@ public class TableRecordReaderImpl {
* @return The getCounter method or null if not available.
* @throws IOException
*/
- private Method retrieveGetCounterWithStringsParams(TaskAttemptContext context)
+ protected static Method retrieveGetCounterWithStringsParams(TaskAttemptContext context)
throws IOException {
Method m = null;
try {
@@ -259,11 +257,6 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- // we can get access to counters only if hbase uses new mapreduce APIs
- if (this.getCounter == null) {
- return;
- }
-
byte[] serializedMetrics = currentScan.getAttribute(
Scan.SCAN_ATTRIBUTES_METRICS_DATA);
if (serializedMetrics == null || serializedMetrics.length == 0 ) {
@@ -272,16 +265,25 @@ public class TableRecordReaderImpl {
ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
+ updateCounters(scanMetrics, numRestarts, getCounter, context);
+ }
+
+ protected static void updateCounters(ScanMetrics scanMetrics, long numScannerRestarts,
+ Method getCounter, TaskAttemptContext context) {
+ // we can get access to counters only if hbase uses new mapreduce APIs
+ if (getCounter == null) {
+ return;
+ }
+
try {
for (Map.Entry<String, Long> entry:scanMetrics.getMetricsMap().entrySet()) {
- Counter ct = (Counter)this.getCounter.invoke(context,
+ Counter ct = (Counter)getCounter.invoke(context,
HBASE_COUNTER_GROUP_NAME, entry.getKey());
ct.increment(entry.getValue());
}
-
- ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
- "NUM_SCANNER_RESTARTS")).increment(numRestarts);
+ ((Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
+ "NUM_SCANNER_RESTARTS")).increment(numScannerRestarts);
} catch (Exception e) {
LOG.debug("can't update counter." + StringUtils.stringifyException(e));
}