You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2015/10/26 18:34:07 UTC
[01/14] hbase git commit: HBASE-14343 Fix debug message in
SimpleRegionNormalizer for small regions (Lars Francke)
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 8b7796b0b -> 899857609
HBASE-14343 Fix debug message in SimpleRegionNormalizer for small regions (Lars Francke)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6875e6d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6875e6d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6875e6d4
Branch: refs/heads/hbase-12439
Commit: 6875e6d4d4a2b57f49febbf7a10776bdf84b7d6b
Parents: 8b7796b
Author: Lars George <la...@apache.org>
Authored: Sat Oct 24 11:19:32 2015 +0200
Committer: Lars George <la...@apache.org>
Committed: Sat Oct 24 11:19:32 2015 +0200
----------------------------------------------------------------------
.../normalizer/SimpleRegionNormalizer.java | 22 +++++++++++---------
1 file changed, 12 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6875e6d4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 8971664..d843bd1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -48,7 +48,9 @@ import java.util.List;
*/
@InterfaceAudience.Private
public class SimpleRegionNormalizer implements RegionNormalizer {
+
private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
+ private static final int MIN_REGION_COUNT = 3;
private MasterServices masterServices;
/**
@@ -68,8 +70,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
* @return normalization plan to execute
*/
@Override
- public NormalizationPlan computePlanForTable(TableName table)
- throws HBaseIOException {
+ public NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException {
if (table == null || table.isSystemTable()) {
LOG.debug("Normalization of table " + table + " isn't allowed");
return EmptyNormalizationPlan.getInstance();
@@ -79,9 +80,10 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
getRegionsOfTable(table);
//TODO: should we make min number of regions a config param?
- if (tableRegions == null || tableRegions.size() < 3) {
- LOG.debug("Table " + table + " has " + tableRegions.size() + " regions, required min number"
- + " of regions for normalizer to run is 3, not running normalizer");
+ if (tableRegions == null || tableRegions.size() < MIN_REGION_COUNT) {
+ int nrRegions = tableRegions == null ? 0 : tableRegions.size();
+ LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
+ + " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
return EmptyNormalizationPlan.getInstance();
}
@@ -93,7 +95,6 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
// A is a smallest region, B is it's smallest neighbor
Pair<HRegionInfo, Long> smallestRegion = new Pair<>();
- Pair<HRegionInfo, Long> smallestNeighborOfSmallestRegion;
int smallestRegionIndex = 0;
for (int i = 0; i < tableRegions.size(); i++) {
@@ -115,16 +116,17 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
// now get smallest neighbor of smallest region
long leftNeighborSize = -1;
- long rightNeighborSize = -1;
if (smallestRegionIndex > 0) {
leftNeighborSize = getRegionSize(tableRegions.get(smallestRegionIndex - 1));
}
+ long rightNeighborSize = -1;
if (smallestRegionIndex < tableRegions.size() - 1) {
rightNeighborSize = getRegionSize(tableRegions.get(smallestRegionIndex + 1));
}
+ Pair<HRegionInfo, Long> smallestNeighborOfSmallestRegion;
if (leftNeighborSize == -1) {
smallestNeighborOfSmallestRegion =
new Pair<>(tableRegions.get(smallestRegionIndex + 1), rightNeighborSize);
@@ -154,11 +156,11 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
+ largestRegion.getSecond() + ", more than 2 times than avg size, splitting");
return new SplitNormalizationPlan(largestRegion.getFirst(), null);
} else {
- if ((smallestRegion.getSecond() + smallestNeighborOfSmallestRegion.getSecond()
- < avgRegionSize)) {
+ if (smallestRegion.getSecond() + smallestNeighborOfSmallestRegion.getSecond()
+ < avgRegionSize) {
LOG.debug("Table " + table + ", smallest region size: " + smallestRegion.getSecond()
+ " and its smallest neighbor size: " + smallestNeighborOfSmallestRegion.getSecond()
- + ", less than half the avg size, merging them");
+ + ", less than the avg size, merging them");
return new MergeNormalizationPlan(smallestRegion.getFirst(),
smallestNeighborOfSmallestRegion.getFirst());
} else {
[05/14] hbase git commit: HBASE-14676 HBaseTestCase clean out: Purge
Incommon Interface and Table and Region implementations
Posted by sy...@apache.org.
HBASE-14676 HBaseTestCase clean out: Purge Incommon Interface and Table and Region implementations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/604ff56e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/604ff56e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/604ff56e
Branch: refs/heads/hbase-12439
Commit: 604ff56ef3c74770a1f0480eebe2d9a45649e1b9
Parents: caae3b2
Author: stack <st...@apache.org>
Authored: Sat Oct 24 10:20:40 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 10:20:52 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HBaseTestCase.java | 244 +------------------
.../apache/hadoop/hbase/TestMultiVersions.java | 15 +-
.../apache/hadoop/hbase/TimestampTestBase.java | 122 +++++-----
.../hbase/regionserver/TestCompaction.java | 15 +-
.../hbase/regionserver/TestMajorCompaction.java | 12 +-
.../hbase/regionserver/TestMinorCompaction.java | 4 +-
.../regionserver/TestMobStoreCompaction.java | 16 +-
.../hadoop/hbase/regionserver/TestScanner.java | 154 ++++++------
8 files changed, 178 insertions(+), 404 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index 597573a..1fb096d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -19,35 +19,29 @@
package org.apache.hadoop.hbase;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
import java.util.NavigableMap;
-import junit.framework.AssertionFailedError;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionAsTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import junit.framework.AssertionFailedError;
+import junit.framework.TestCase;
+
/**
* Abstract HBase test class. Initializes a few things that can come in handly
* like an HBaseConfiguration and filesystem.
@@ -239,7 +233,7 @@ public abstract class HBaseTestCase extends TestCase {
if (startKeyBytes == null || startKeyBytes.length == 0) {
startKeyBytes = START_KEY_BYTES;
}
- return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
+ return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
startKeyBytes, endKey, -1);
}
@@ -251,18 +245,15 @@ public abstract class HBaseTestCase extends TestCase {
* Add content to region <code>r</code> on the passed column
* <code>column</code>.
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
- * @param updater An instance of {@link Incommon}.
- * @param columnFamily
- * @param writeToWAL
* @throws IOException
* @return count of what we added.
*/
- public static long addContent(final Incommon updater,
+ public static long addContent(final Table updater,
final String columnFamily) throws IOException {
return addContent(updater, columnFamily, START_KEY_BYTES, null);
}
- public static long addContent(final Incommon updater, final String family,
+ public static long addContent(final Table updater, final String family,
final String column) throws IOException {
return addContent(updater, family, column, START_KEY_BYTES, null);
}
@@ -271,21 +262,16 @@ public abstract class HBaseTestCase extends TestCase {
* Add content to region <code>r</code> on the passed column
* <code>column</code>.
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
- * @param updater An instance of {@link Incommon}.
- * @param columnFamily
- * @param startKeyBytes Where to start the rows inserted
- * @param endKey Where to stop inserting rows.
- * @param writeToWAL
* @return count of what we added.
* @throws IOException
*/
- public static long addContent(final Incommon updater, final String columnFamily,
+ public static long addContent(final Table updater, final String columnFamily,
final byte [] startKeyBytes, final byte [] endKey)
throws IOException {
return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
}
- public static long addContent(final Incommon updater, final String family, String column,
+ public static long addContent(final Table updater, final String family, String column,
final byte [] startKeyBytes, final byte [] endKey) throws IOException {
return addContent(updater, family, column, startKeyBytes, endKey, -1);
}
@@ -294,16 +280,10 @@ public abstract class HBaseTestCase extends TestCase {
* Add content to region <code>r</code> on the passed column
* <code>column</code>.
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
- * @param updater An instance of {@link Incommon}.
- * @param column
- * @param startKeyBytes Where to start the rows inserted
- * @param endKey Where to stop inserting rows.
- * @param ts Timestamp to write the content with.
- * @param writeToWAL
* @return count of what we added.
* @throws IOException
*/
- public static long addContent(final Incommon updater,
+ public static long addContent(final Table updater,
final String columnFamily,
final String column,
final byte [] startKeyBytes, final byte [] endKey, final long ts)
@@ -378,209 +358,6 @@ public abstract class HBaseTestCase extends TestCase {
return count;
}
- /**
- * Implementors can flushcache.
- */
- public interface FlushCache {
- /**
- * @throws IOException
- */
- void flushcache() throws IOException;
- }
-
- /**
- * Interface used by tests so can do common operations against an HTable
- * or an HRegion.
- *
- * TOOD: Come up w/ a better name for this interface.
- */
- public interface Incommon {
- /**
- *
- * @param delete
- * @param writeToWAL
- * @throws IOException
- */
- void delete(Delete delete, boolean writeToWAL)
- throws IOException;
-
- /**
- * @param put
- * @throws IOException
- */
- void put(Put put) throws IOException;
-
- Result get(Get get) throws IOException;
-
- /**
- * @param family
- * @param qualifiers
- * @param firstRow
- * @param ts
- * @return scanner for specified columns, first row and timestamp
- * @throws IOException
- */
- ScannerIncommon getScanner(
- byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
- )
- throws IOException;
- }
-
- /**
- * A class that makes a {@link Incommon} out of a {@link HRegion}
- */
- public static class HRegionIncommon implements Incommon, FlushCache {
- final HRegion region;
-
- /**
- * @param HRegion
- */
- public HRegionIncommon(final HRegion HRegion) {
- this.region = HRegion;
- }
-
- public HRegionIncommon(final Region region) {
- this.region = (HRegion)region;
- }
-
- public void put(Put put) throws IOException {
- region.put(put);
- }
-
- public void delete(Delete delete, boolean writeToWAL)
- throws IOException {
- this.region.delete(delete);
- }
-
- public Result get(Get get) throws IOException {
- return region.get(get);
- }
-
- public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
- byte [] firstRow, long ts)
- throws IOException {
- Scan scan = new Scan(firstRow);
- if(qualifiers == null || qualifiers.length == 0) {
- scan.addFamily(family);
- } else {
- for(int i=0; i<qualifiers.length; i++){
- scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
- }
- }
- scan.setTimeRange(0, ts);
- return new
- InternalScannerIncommon(region.getScanner(scan));
- }
-
- public void flushcache() throws IOException {
- this.region.flush(true);
- }
- }
-
- /**
- * A class that makes a {@link Incommon} out of a {@link Table}
- */
- public static class HTableIncommon implements Incommon {
- final Table table;
-
- /**
- * @param table
- */
- public HTableIncommon(final Table table) {
- super();
- this.table = table;
- }
-
- public void put(Put put) throws IOException {
- table.put(put);
- }
-
-
- public void delete(Delete delete, boolean writeToWAL)
- throws IOException {
- this.table.delete(delete);
- }
-
- public Result get(Get get) throws IOException {
- return table.get(get);
- }
-
- public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
- byte [] firstRow, long ts)
- throws IOException {
- Scan scan = new Scan(firstRow);
- if(qualifiers == null || qualifiers.length == 0) {
- scan.addFamily(family);
- } else {
- for(int i=0; i<qualifiers.length; i++){
- scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
- }
- }
- scan.setTimeRange(0, ts);
- return new
- ClientScannerIncommon(table.getScanner(scan));
- }
- }
-
- public interface ScannerIncommon
- extends Iterable<Result> {
- boolean next(List<Cell> values)
- throws IOException;
-
- void close() throws IOException;
- }
-
- public static class ClientScannerIncommon implements ScannerIncommon {
- ResultScanner scanner;
- public ClientScannerIncommon(ResultScanner scanner) {
- this.scanner = scanner;
- }
-
- @Override
- public boolean next(List<Cell> values)
- throws IOException {
- Result results = scanner.next();
- if (results == null) {
- return false;
- }
- values.clear();
- values.addAll(results.listCells());
- return true;
- }
-
- public void close() throws IOException {
- scanner.close();
- }
-
- public Iterator<Result> iterator() {
- return scanner.iterator();
- }
- }
-
- public static class InternalScannerIncommon implements ScannerIncommon {
- InternalScanner scanner;
-
- public InternalScannerIncommon(InternalScanner scanner) {
- this.scanner = scanner;
- }
-
- @Override
- public boolean next(List<Cell> results)
- throws IOException {
- return scanner.next(results);
- }
-
- @Override
- public void close() throws IOException {
- scanner.close();
- }
-
- @Override
- public Iterator<Result> iterator() {
- throw new UnsupportedOperationException();
- }
- }
-
protected void assertResultEquals(final HRegion region, final byte [] row,
final byte [] family, final byte [] qualifier, final long timestamp,
final byte [] value)
@@ -669,5 +446,4 @@ public abstract class HBaseTestCase extends TestCase {
Bytes.toStringBinary(actual) + ">");
}
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
index 278973e..439e722 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
@@ -30,9 +30,7 @@ import java.util.NavigableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestCase.FlushCache;
-import org.apache.hadoop.hbase.HBaseTestCase.HTableIncommon;
-import org.apache.hadoop.hbase.HBaseTestCase.Incommon;
+import org.apache.hadoop.hbase.TimestampTestBase.FlushCache;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
@@ -97,8 +95,7 @@ public class TestMultiVersions {
Table table = UTIL.getConnection().getTable(desc.getTableName());
// TODO: Remove these deprecated classes or pull them in here if this is
// only test using them.
- Incommon incommon = new HTableIncommon(table);
- TimestampTestBase.doTestDelete(incommon, new FlushCache() {
+ TimestampTestBase.doTestDelete(table, new FlushCache() {
public void flushcache() throws IOException {
UTIL.getHBaseCluster().flushcache();
}
@@ -106,7 +103,7 @@ public class TestMultiVersions {
// Perhaps drop and readd the table between tests so the former does
// not pollute this latter? Or put into separate tests.
- TimestampTestBase.doTestTimestampScanning(incommon, new FlushCache() {
+ TimestampTestBase.doTestTimestampScanning(table, new FlushCache() {
public void flushcache() throws IOException {
UTIL.getMiniHBaseCluster().flushcache();
}
@@ -136,7 +133,7 @@ public class TestMultiVersions {
desc.addFamily(hcd);
this.admin.createTable(desc);
Put put = new Put(row, timestamp1);
- put.add(contents, contents, value1);
+ put.addColumn(contents, contents, value1);
Table table = UTIL.getConnection().getTable(desc.getTableName());
table.put(put);
// Shut down and restart the HBase cluster
@@ -148,7 +145,7 @@ public class TestMultiVersions {
table = UTIL.getConnection().getTable(desc.getTableName());
// Overwrite previous value
put = new Put(row, timestamp2);
- put.add(contents, contents, value2);
+ put.addColumn(contents, contents, value2);
table.put(put);
// Now verify that getRow(row, column, latest) works
Get get = new Get(row);
@@ -221,7 +218,7 @@ public class TestMultiVersions {
for (int i = 0; i < startKeys.length; i++) {
for (int j = 0; j < timestamp.length; j++) {
Put put = new Put(rows[i], timestamp[j]);
- put.add(HConstants.CATALOG_FAMILY, null, timestamp[j],
+ put.addColumn(HConstants.CATALOG_FAMILY, null, timestamp[j],
Bytes.toBytes(timestamp[j]));
puts.add(put);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java
index 40baf41..8e8d4ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java
@@ -24,15 +24,19 @@ import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
/**
* Tests user specifiable time stamps putting, getting and scanning. Also
* tests same in presence of deletes. Test cores are written so can be
* run against an HRegion and against an HTable: i.e. both local and remote.
*/
-public class TimestampTestBase extends HBaseTestCase {
+public class TimestampTestBase {
private static final long T0 = 10L;
private static final long T1 = 100L;
private static final long T2 = 200L;
@@ -42,73 +46,73 @@ public class TimestampTestBase extends HBaseTestCase {
private static final byte [] ROW = Bytes.toBytes("row");
- /*
+ interface FlushCache {
+ void flushcache() throws IOException;
+ }
+
+ /*
* Run test that delete works according to description in <a
* href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
* @param incommon
* @param flusher
* @throws IOException
*/
- public static void doTestDelete(final Incommon incommon, FlushCache flusher)
+ public static void doTestDelete(final Table table, FlushCache flusher)
throws IOException {
// Add values at various timestamps (Values are timestampes as bytes).
- put(incommon, T0);
- put(incommon, T1);
- put(incommon, T2);
- put(incommon);
+ put(table, T0);
+ put(table, T1);
+ put(table, T2);
+ put(table);
// Verify that returned versions match passed timestamps.
- assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
+ assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
- // If I delete w/o specifying a timestamp, this means I'm deleting the
- // latest.
- delete(incommon);
- // Verify that I get back T2 through T1 -- that the latest version has
- // been deleted.
- assertVersions(incommon, new long [] {T2, T1, T0});
+ // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
+ delete(table);
+ // Verify that I get back T2 through T1 -- that the latest version has been deleted.
+ assertVersions(table, new long [] {T2, T1, T0});
// Flush everything out to disk and then retry
flusher.flushcache();
- assertVersions(incommon, new long [] {T2, T1, T0});
+ assertVersions(table, new long [] {T2, T1, T0});
// Now add, back a latest so I can test remove other than the latest.
- put(incommon);
- assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
- delete(incommon, T2);
- assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
+ put(table);
+ assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
+ delete(table, T2);
+ assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
// Flush everything out to disk and then retry
flusher.flushcache();
- assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
+ assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
// Now try deleting all from T2 back inclusive (We first need to add T2
- // back into the mix and to make things a little interesting, delete and
- // then readd T1.
- put(incommon, T2);
- delete(incommon, T1);
- put(incommon, T1);
+ // back into the mix and to make things a little interesting, delete and then readd T1.
+ put(table, T2);
+ delete(table, T1);
+ put(table, T1);
Delete delete = new Delete(ROW);
- delete.deleteColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
- incommon.delete(delete, true);
+ delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
+ table.delete(delete);
// Should only be current value in set. Assert this is so
- assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
+ assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
// Flush everything out to disk and then redo above tests
flusher.flushcache();
- assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
+ assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
}
- private static void assertOnlyLatest(final Incommon incommon,
- final long currentTime)
+ private static void assertOnlyLatest(final Table incommon, final long currentTime)
throws IOException {
Get get = null;
get = new Get(ROW);
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
get.setMaxVersions(3);
Result result = incommon.get(get);
- assertEquals(1, result.size());
+ Assert.assertEquals(1, result.size());
long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
- assertEquals(time, currentTime);
+ Assert.assertEquals(time, currentTime);
}
/*
@@ -119,7 +123,7 @@ public class TimestampTestBase extends HBaseTestCase {
* @param tss
* @throws IOException
*/
- public static void assertVersions(final Incommon incommon, final long [] tss)
+ public static void assertVersions(final Table incommon, final long [] tss)
throws IOException {
// Assert that 'latest' is what we expect.
Get get = null;
@@ -128,7 +132,7 @@ public class TimestampTestBase extends HBaseTestCase {
Result r = incommon.get(get);
byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
long t = Bytes.toLong(bytes);
- assertEquals(tss[0], t);
+ Assert.assertEquals(tss[0], t);
// Now assert that if we ask for multiple versions, that they come out in
// order.
@@ -137,10 +141,10 @@ public class TimestampTestBase extends HBaseTestCase {
get.setMaxVersions(tss.length);
Result result = incommon.get(get);
Cell [] kvs = result.rawCells();
- assertEquals(kvs.length, tss.length);
+ Assert.assertEquals(kvs.length, tss.length);
for(int i=0;i<kvs.length;i++) {
t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
- assertEquals(tss[i], t);
+ Assert.assertEquals(tss[i], t);
}
// Determine highest stamp to set as next max stamp
@@ -153,10 +157,10 @@ public class TimestampTestBase extends HBaseTestCase {
get.setMaxVersions(kvs.length - 1);
result = incommon.get(get);
kvs = result.rawCells();
- assertEquals(kvs.length, tss.length - 1);
+ Assert.assertEquals(kvs.length, tss.length - 1);
for(int i=1;i<kvs.length;i++) {
t = Bytes.toLong(CellUtil.cloneValue(kvs[i-1]));
- assertEquals(tss[i], t);
+ Assert.assertEquals(tss[i], t);
}
// Test scanner returns expected version
@@ -169,7 +173,7 @@ public class TimestampTestBase extends HBaseTestCase {
* @param flusher
* @throws IOException
*/
- public static void doTestTimestampScanning(final Incommon incommon,
+ public static void doTestTimestampScanning(final Table incommon,
final FlushCache flusher)
throws IOException {
// Add a couple of values for three different timestamps.
@@ -180,12 +184,12 @@ public class TimestampTestBase extends HBaseTestCase {
int count = assertScanContentTimestamp(incommon,
HConstants.LATEST_TIMESTAMP);
// Assert I get same count when I scan at each timestamp.
- assertEquals(count, assertScanContentTimestamp(incommon, T0));
- assertEquals(count, assertScanContentTimestamp(incommon, T1));
+ Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
+ Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
// Flush everything out to disk and then retry
flusher.flushcache();
- assertEquals(count, assertScanContentTimestamp(incommon, T0));
- assertEquals(count, assertScanContentTimestamp(incommon, T1));
+ Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
+ Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
}
/*
@@ -195,10 +199,12 @@ public class TimestampTestBase extends HBaseTestCase {
* @return Count of items scanned.
* @throws IOException
*/
- public static int assertScanContentTimestamp(final Incommon in, final long ts)
+ public static int assertScanContentTimestamp(final Table in, final long ts)
throws IOException {
- ScannerIncommon scanner =
- in.getScanner(COLUMNS[0], null, HConstants.EMPTY_START_ROW, ts);
+ Scan scan = new Scan(HConstants.EMPTY_START_ROW);
+ scan.addFamily(FAMILY_NAME);
+ scan.setTimeRange(0, ts);
+ ResultScanner scanner = in.getScanner(scan);
int count = 0;
try {
// TODO FIX
@@ -221,12 +227,12 @@ public class TimestampTestBase extends HBaseTestCase {
return count;
}
- public static void put(final Incommon loader, final long ts)
+ public static void put(final Table loader, final long ts)
throws IOException {
put(loader, Bytes.toBytes(ts), ts);
}
- public static void put(final Incommon loader)
+ public static void put(final Table loader)
throws IOException {
long ts = HConstants.LATEST_TIMESTAMP;
put(loader, Bytes.toBytes(ts), ts);
@@ -239,39 +245,39 @@ public class TimestampTestBase extends HBaseTestCase {
* @param ts
* @throws IOException
*/
- public static void put(final Incommon loader, final byte [] bytes,
+ public static void put(final Table loader, final byte [] bytes,
final long ts)
throws IOException {
Put put = new Put(ROW, ts);
put.setDurability(Durability.SKIP_WAL);
- put.add(FAMILY_NAME, QUALIFIER_NAME, bytes);
+ put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
loader.put(put);
}
- public static void delete(final Incommon loader) throws IOException {
+ public static void delete(final Table loader) throws IOException {
delete(loader, null);
}
- public static void delete(final Incommon loader, final byte [] column)
+ public static void delete(final Table loader, final byte [] column)
throws IOException {
delete(loader, column, HConstants.LATEST_TIMESTAMP);
}
- public static void delete(final Incommon loader, final long ts)
+ public static void delete(final Table loader, final long ts)
throws IOException {
delete(loader, null, ts);
}
- public static void delete(final Incommon loader, final byte [] column,
+ public static void delete(final Table loader, final byte [] column,
final long ts)
throws IOException {
Delete delete = ts == HConstants.LATEST_TIMESTAMP?
new Delete(ROW): new Delete(ROW, ts);
- delete.deleteColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
- loader.delete(delete, true);
+ delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
+ loader.delete(delete);
}
- public static Result get(final Incommon loader) throws IOException {
+ public static Result get(final Table loader) throws IOException {
return loader.get(new Get(ROW));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 0460972..54bf60e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -37,8 +37,6 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -47,7 +45,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -57,6 +54,7 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -86,7 +84,6 @@ import org.mockito.stubbing.Answer;
@Category({RegionServerTests.class, MediumTests.class})
public class TestCompaction {
@Rule public TestName name = new TestName();
- private static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
protected Configuration conf = UTIL.getConfiguration();
@@ -148,15 +145,15 @@ public class TestCompaction {
int jmax = (int) Math.ceil(15.0/compactionThreshold);
byte [] pad = new byte[1000]; // 1 KB chunk
for (int i = 0; i < compactionThreshold; i++) {
- HRegionIncommon loader = new HRegionIncommon(r);
+ Table loader = new RegionAsTable(r);
Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
p.setDurability(Durability.SKIP_WAL);
for (int j = 0; j < jmax; j++) {
- p.add(COLUMN_FAMILY, Bytes.toBytes(j), pad);
+ p.addColumn(COLUMN_FAMILY, Bytes.toBytes(j), pad);
}
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY));
loader.put(p);
- loader.flushcache();
+ r.flush(true);
}
HRegion spyR = spy(r);
@@ -230,9 +227,9 @@ public class TestCompaction {
}
private void createStoreFile(final HRegion region, String family) throws IOException {
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
HBaseTestCase.addContent(loader, family);
- loader.flushcache();
+ region.flush(true);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 0409df3..ddd00d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -48,6 +47,7 @@ import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
@@ -198,7 +198,7 @@ public class TestMajorCompaction {
createStoreFile(r);
}
// Add more content.
- HBaseTestCase.addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY));
+ HBaseTestCase.addContent(new RegionAsTable(r), Bytes.toString(COLUMN_FAMILY));
// Now there are about 5 versions of each column.
// Default is that there only 3 (MAXVERSIONS) versions allowed per column.
@@ -387,16 +387,16 @@ public class TestMajorCompaction {
}
private void createStoreFile(final Region region, String family) throws IOException {
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
HBaseTestCase.addContent(loader, family);
- loader.flushcache();
+ region.flush(true);
}
private void createSmallerStoreFile(final Region region) throws IOException {
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
"bbb").getBytes(), null);
- loader.flushcache();
+ region.flush(true);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
index 47f3a8f..261c007 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
@@ -28,7 +28,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -37,6 +36,7 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
@@ -162,7 +162,7 @@ public class TestMinorCompaction {
}
private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception {
- HRegionIncommon loader = new HRegionIncommon(r);
+ Table loader = new RegionAsTable(r);
for (int i = 0; i < compactionThreshold + 1; i++) {
HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
thirdRowBytes, i);
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
index 8d2d857..a01cc66 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -125,12 +125,12 @@ public class TestMobStoreCompaction {
public void testSmallerValue() throws Exception {
init(UTIL.getConfiguration(), 500);
byte[] dummyData = makeDummyData(300); // smaller than mob threshold
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
// one hfile per row
for (int i = 0; i < compactionThreshold; i++) {
Put p = createPut(i, dummyData);
loader.put(p);
- loader.flushcache();
+ region.flush(true);
}
assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
assertEquals("Before compaction: mob file count", 0, countMobFiles());
@@ -153,11 +153,11 @@ public class TestMobStoreCompaction {
public void testLargerValue() throws Exception {
init(UTIL.getConfiguration(), 200);
byte[] dummyData = makeDummyData(300); // larger than mob threshold
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
for (int i = 0; i < compactionThreshold; i++) {
Put p = createPut(i, dummyData);
loader.put(p);
- loader.flushcache();
+ region.flush(true);
}
assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
@@ -221,14 +221,14 @@ public class TestMobStoreCompaction {
public void testMajorCompactionAfterDelete() throws Exception {
init(UTIL.getConfiguration(), 100);
byte[] dummyData = makeDummyData(200); // larger than mob threshold
- HRegionIncommon loader = new HRegionIncommon(region);
+ Table loader = new RegionAsTable(region);
// create hfiles and mob hfiles but don't trigger compaction
int numHfiles = compactionThreshold - 1;
byte[] deleteRow = Bytes.add(STARTROW, Bytes.toBytes(0));
for (int i = 0; i < numHfiles; i++) {
Put p = createPut(i, dummyData);
loader.put(p);
- loader.flushcache();
+ region.flush(true);
}
assertEquals("Before compaction: store files", numHfiles, countStoreFiles());
assertEquals("Before compaction: mob file count", numHfiles, countMobFiles());
@@ -239,7 +239,7 @@ public class TestMobStoreCompaction {
Delete delete = new Delete(deleteRow);
delete.addFamily(COLUMN_FAMILY);
region.delete(delete);
- loader.flushcache();
+ region.flush(true);
assertEquals("Before compaction: store files", numHfiles + 1, countStoreFiles());
assertEquals("Before compaction: mob files", numHfiles, countMobFiles());
http://git-wip-us.apache.org/repos/asf/hbase/blob/604ff56e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
index cdef351..ad71bc7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
@@ -36,8 +36,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
-import org.apache.hadoop.hbase.HBaseTestCase.ScannerIncommon;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -49,7 +47,9 @@ import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
@@ -99,11 +99,10 @@ public class TestScanner {
private static final long START_CODE = Long.MAX_VALUE;
- private Region r;
- private HRegionIncommon region;
+ private HRegion region;
private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
- final private byte[] col1, col2;
+ final private byte[] col1;
public TestScanner() {
super();
@@ -115,7 +114,6 @@ public class TestScanner {
thirdRowBytes = START_KEY_BYTES.clone();
thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
col1 = Bytes.toBytes("column1");
- col2 = Bytes.toBytes("column2");
}
/**
@@ -127,14 +125,14 @@ public class TestScanner {
byte [] startrow = Bytes.toBytes("bbb");
byte [] stoprow = Bytes.toBytes("ccc");
try {
- this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
+ this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
List<Cell> results = new ArrayList<Cell>();
// Do simple test of getting one row only first.
Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
scan.addFamily(HConstants.CATALOG_FAMILY);
- InternalScanner s = r.getScanner(scan);
+ InternalScanner s = region.getScanner(scan);
int count = 0;
while (s.next(results)) {
count++;
@@ -145,7 +143,7 @@ public class TestScanner {
scan = new Scan(startrow, stoprow);
scan.addFamily(HConstants.CATALOG_FAMILY);
- s = r.getScanner(scan);
+ s = region.getScanner(scan);
count = 0;
Cell kv = null;
results = new ArrayList<Cell>();
@@ -162,14 +160,14 @@ public class TestScanner {
assertTrue(count > 10);
s.close();
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
void rowPrefixFilter(Scan scan) throws IOException {
List<Cell> results = new ArrayList<Cell>();
scan.addFamily(HConstants.CATALOG_FAMILY);
- InternalScanner s = r.getScanner(scan);
+ InternalScanner s = region.getScanner(scan);
boolean hasMore = true;
while (hasMore) {
hasMore = s.next(results);
@@ -185,7 +183,7 @@ public class TestScanner {
void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
List<Cell> results = new ArrayList<Cell>();
scan.addFamily(HConstants.CATALOG_FAMILY);
- InternalScanner s = r.getScanner(scan);
+ InternalScanner s = region.getScanner(scan);
boolean hasMore = true;
while (hasMore) {
hasMore = s.next(results);
@@ -200,8 +198,8 @@ public class TestScanner {
@Test
public void testFilters() throws IOException {
try {
- this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
+ this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
byte [] prefix = Bytes.toBytes("ab");
Filter newFilter = new PrefixFilter(prefix);
Scan scan = new Scan();
@@ -215,7 +213,7 @@ public class TestScanner {
rowInclusiveStopFilter(scan, stopRow);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -227,10 +225,10 @@ public class TestScanner {
@Test
public void testRaceBetweenClientAndTimeout() throws Exception {
try {
- this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
+ this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
Scan scan = new Scan();
- InternalScanner s = r.getScanner(scan);
+ InternalScanner s = region.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
try {
s.next(results);
@@ -242,7 +240,7 @@ public class TestScanner {
return;
}
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -252,71 +250,70 @@ public class TestScanner {
@Test
public void testScanner() throws IOException {
try {
- r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- region = new HRegionIncommon(r);
+ region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ Table table = new RegionAsTable(region);
// Write information to the meta table
Put put = new Put(ROW_KEY, System.currentTimeMillis());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ put.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
REGION_INFO.toByteArray());
- region.put(put);
+ table.put(put);
// What we just committed is in the memstore. Verify that we can get
// it back both with scanning and get
scan(false, null);
- getRegionInfo();
+ getRegionInfo(table);
// Close and re-open
- ((HRegion)r).close();
- r = HRegion.openHRegion(r, null);
- region = new HRegionIncommon(r);
+ ((HRegion)region).close();
+ region = HRegion.openHRegion(region, null);
+ table = new RegionAsTable(region);
// Verify we can get the data back now that it is on disk.
scan(false, null);
- getRegionInfo();
+ getRegionInfo(table);
// Store some new information
String address = HConstants.LOCALHOST_IP + ":" + HBaseTestingUtility.randomFreePort();
put = new Put(ROW_KEY, System.currentTimeMillis());
- put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ put.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
Bytes.toBytes(address));
// put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE));
- region.put(put);
+ table.put(put);
// Validate that we can still get the HRegionInfo, even though it is in
// an older row on disk and there is a newer row in the memstore
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
// flush cache
-
- region.flushcache();
+ this.region.flush(true);
// Validate again
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
// Close and reopen
- ((HRegion)r).close();
- r = HRegion.openHRegion(r,null);
- region = new HRegionIncommon(r);
+ ((HRegion)region).close();
+ region = HRegion.openHRegion(region,null);
+ table = new RegionAsTable(region);
// Validate again
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
// Now update the information again
@@ -324,38 +321,37 @@ public class TestScanner {
put = new Put(ROW_KEY, System.currentTimeMillis());
- put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
- Bytes.toBytes(address));
- region.put(put);
+ put.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(address));
+ table.put(put);
// Validate again
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
// flush cache
- region.flushcache();
+ region.flush(true);
// Validate again
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
// Close and reopen
- ((HRegion)r).close();
- r = HRegion.openHRegion(r,null);
- region = new HRegionIncommon(r);
+ ((HRegion)this.region).close();
+ this.region = HRegion.openHRegion(region, null);
+ table = new RegionAsTable(this.region);
// Validate again
scan(true, address.toString());
- getRegionInfo();
+ getRegionInfo(table);
} finally {
// clean up
- HBaseTestingUtility.closeRegionAndWAL(r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -387,7 +383,7 @@ public class TestScanner {
for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
}
- scanner = r.getScanner(scan);
+ scanner = region.getScanner(scan);
while (scanner.next(results)) {
assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER));
@@ -448,10 +444,10 @@ public class TestScanner {
/** Use get to retrieve the HRegionInfo and validate it */
- private void getRegionInfo() throws IOException {
+ private void getRegionInfo(Table table) throws IOException {
Get get = new Get(ROW_KEY);
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
- Result result = region.get(get);
+ Result result = table.get(get);
byte [] bytes = result.value();
validateRegionInfo(bytes);
}
@@ -464,10 +460,11 @@ public class TestScanner {
*/
@Test
public void testScanAndSyncFlush() throws Exception {
- this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- HRegionIncommon hri = new HRegionIncommon(r);
+ this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ Table hri = new RegionAsTable(region);
try {
- LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
+ LOG.info("Added: " +
+ HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
int count = count(hri, -1, false);
assertEquals(count, count(hri, 100, false)); // do a sync flush.
@@ -475,7 +472,7 @@ public class TestScanner {
LOG.error("Failed", e);
throw e;
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -487,10 +484,11 @@ public class TestScanner {
*/
@Test
public void testScanAndRealConcurrentFlush() throws Exception {
- this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
- HRegionIncommon hri = new HRegionIncommon(r);
+ this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
+ Table hri = new RegionAsTable(region);
try {
- LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
+ LOG.info("Added: " +
+ HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
int count = count(hri, -1, false);
assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush
@@ -498,7 +496,7 @@ public class TestScanner {
LOG.error("Failed", e);
throw e;
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -512,8 +510,8 @@ public class TestScanner {
@SuppressWarnings("deprecation")
public void testScanAndConcurrentMajorCompact() throws Exception {
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name.getMethodName());
- this.r = TEST_UTIL.createLocalHRegion(htd, null, null);
- HRegionIncommon hri = new HRegionIncommon(r);
+ this.region = TEST_UTIL.createLocalHRegion(htd, null, null);
+ Table hri = new RegionAsTable(region);
try {
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
@@ -524,18 +522,18 @@ public class TestScanner {
Delete dc = new Delete(firstRowBytes);
/* delete column1 of firstRow */
dc.deleteColumns(fam1, col1);
- r.delete(dc);
- r.flush(true);
+ region.delete(dc);
+ region.flush(true);
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
secondRowBytes, thirdRowBytes);
HBaseTestCase.addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
secondRowBytes, thirdRowBytes);
- r.flush(true);
+ region.flush(true);
- InternalScanner s = r.getScanner(new Scan());
+ InternalScanner s = region.getScanner(new Scan());
// run a major compact, column1 of firstRow will be cleaned.
- r.compact(true);
+ region.compact(true);
List<Cell> results = new ArrayList<Cell>();
s.next(results);
@@ -555,7 +553,7 @@ public class TestScanner {
assertTrue(CellUtil.matchingFamily(results.get(0), fam1));
assertTrue(CellUtil.matchingFamily(results.get(1), fam2));
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.r);
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
}
}
@@ -567,19 +565,20 @@ public class TestScanner {
* @return Count of rows found.
* @throws IOException
*/
- private int count(final HRegionIncommon hri, final int flushIndex,
- boolean concurrent)
+ private int count(final Table countTable, final int flushIndex, boolean concurrent)
throws IOException {
LOG.info("Taking out counting scan");
- ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
- HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
- List<Cell> values = new ArrayList<Cell>();
+ Scan scan = new Scan();
+ for (byte [] qualifier: EXPLICIT_COLS) {
+ scan.addColumn(HConstants.CATALOG_FAMILY, qualifier);
+ }
+ ResultScanner s = countTable.getScanner(scan);
int count = 0;
boolean justFlushed = false;
- while (s.next(values)) {
+ while (s.next() != null) {
if (justFlushed) {
LOG.info("after next() just after next flush");
- justFlushed=false;
+ justFlushed = false;
}
count++;
if (flushIndex == count) {
@@ -587,7 +586,7 @@ public class TestScanner {
Thread t = new Thread() {
public void run() {
try {
- hri.flushcache();
+ region.flush(true);
LOG.info("Finishing flush");
} catch (IOException e) {
LOG.info("Failed flush cache");
@@ -607,5 +606,4 @@ public class TestScanner {
LOG.info("Found " + count + " items");
return count;
}
-
}
[11/14] hbase git commit: HBASE-14624 BucketCache.freeBlock is too
expensive
Posted by sy...@apache.org.
HBASE-14624 BucketCache.freeBlock is too expensive
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/95978477
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/95978477
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/95978477
Branch: refs/heads/hbase-12439
Commit: 95978477f8565defb3c4792f901e833abf63f768
Parents: f8528f6
Author: tedyu <yu...@gmail.com>
Authored: Sat Oct 24 18:45:21 2015 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Sat Oct 24 18:45:21 2015 -0700
----------------------------------------------------------------------
.../hbase/io/hfile/bucket/BucketAllocator.java | 36 +++++++++++---------
1 file changed, 19 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/95978477/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
index faa8724..1e8cb7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
@@ -21,11 +21,10 @@
package org.apache.hadoop.hbase.io.hfile.bucket;
import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.collections.map.LinkedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -173,22 +172,22 @@ public final class BucketAllocator {
final class BucketSizeInfo {
// Free bucket means it has space to allocate a block;
// Completely free bucket means it has no block.
- private List<Bucket> bucketList, freeBuckets, completelyFreeBuckets;
+ private LinkedMap bucketList, freeBuckets, completelyFreeBuckets;
private int sizeIndex;
BucketSizeInfo(int sizeIndex) {
- bucketList = new LinkedList<Bucket>();
- freeBuckets = new LinkedList<Bucket>();
- completelyFreeBuckets = new LinkedList<Bucket>();
+ bucketList = new LinkedMap();
+ freeBuckets = new LinkedMap();
+ completelyFreeBuckets = new LinkedMap();
this.sizeIndex = sizeIndex;
}
public synchronized void instantiateBucket(Bucket b) {
assert b.isUninstantiated() || b.isCompletelyFree();
b.reconfigure(sizeIndex, bucketSizes, bucketCapacity);
- bucketList.add(b);
- freeBuckets.add(b);
- completelyFreeBuckets.add(b);
+ bucketList.put(b, b);
+ freeBuckets.put(b, b);
+ completelyFreeBuckets.put(b, b);
}
public int sizeIndex() {
@@ -201,8 +200,10 @@ public final class BucketAllocator {
*/
public long allocateBlock() {
Bucket b = null;
- if (freeBuckets.size() > 0) // Use up an existing one first...
- b = freeBuckets.get(freeBuckets.size() - 1);
+ if (freeBuckets.size() > 0) {
+ // Use up an existing one first...
+ b = (Bucket) freeBuckets.lastKey();
+ }
if (b == null) {
b = grabGlobalCompletelyFreeBucket();
if (b != null) instantiateBucket(b);
@@ -227,7 +228,7 @@ public final class BucketAllocator {
}
if (completelyFreeBuckets.size() > 0) {
- b = completelyFreeBuckets.get(0);
+ b = (Bucket) completelyFreeBuckets.firstKey();
removeBucket(b);
}
return b;
@@ -241,17 +242,18 @@ public final class BucketAllocator {
}
public void freeBlock(Bucket b, long offset) {
- assert bucketList.contains(b);
+ assert bucketList.containsKey(b);
// else we shouldn't have anything to free...
- assert (!completelyFreeBuckets.contains(b));
+ assert (!completelyFreeBuckets.containsKey(b));
b.free(offset);
- if (!freeBuckets.contains(b)) freeBuckets.add(b);
- if (b.isCompletelyFree()) completelyFreeBuckets.add(b);
+ if (!freeBuckets.containsKey(b)) freeBuckets.put(b, b);
+ if (b.isCompletelyFree()) completelyFreeBuckets.put(b, b);
}
public synchronized IndexStatistics statistics() {
long free = 0, used = 0;
- for (Bucket b : bucketList) {
+ for (Object obj : bucketList.keySet()) {
+ Bucket b = (Bucket) obj;
free += b.freeCount();
used += b.usedCount();
}
[08/14] hbase git commit: HBASE-14678 Experiment: Temporarily disable
balancer and a few others to see if root of crashed/timedout JVMs;
ADD TestDistributedLogSplitting to the mix;
ADDENDUM added TestSnapshotCloneIndependence to removed set of dodgy test
Posted by sy...@apache.org.
HBASE-14678 Experiment: Temporarily disable balancer and a few others to see if root of crashed/timedout JVMs; ADD TestDistributedLogSplitting to the mix; ADDENDUM added TestSnapshotCloneIndependence to removed set of dodgy tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df36aef2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df36aef2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df36aef2
Branch: refs/heads/hbase-12439
Commit: df36aef23c5a4593a3160eb3937c54baf27991d1
Parents: 80a6fd0
Author: stack <st...@apache.org>
Authored: Sat Oct 24 14:26:39 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 14:26:39 2015 -0700
----------------------------------------------------------------------
.../client/TestSnapshotCloneIndependence.java | 481 -------------------
1 file changed, 481 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/df36aef2/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
deleted file mode 100644
index 5f20790..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
+++ /dev/null
@@ -1,481 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.client;
-
-import java.io.IOException;
-import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test to verify that the cloned table is independent of the table from which it was cloned
- */
-@Category({LargeTests.class, ClientTests.class})
-public class TestSnapshotCloneIndependence {
- private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
-
- protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- protected static final int NUM_RS = 2;
- private static final String STRING_TABLE_NAME = "test";
- private static final String TEST_FAM_STR = "fam";
- protected static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
- protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
- private static final int CLEANER_INTERVAL = 100;
-
- /**
- * Setup the config for the cluster and start it
- * @throws Exception on fOailure
- */
- @BeforeClass
- public static void setupCluster() throws Exception {
- setupConf(UTIL.getConfiguration());
- UTIL.startMiniCluster(NUM_RS);
- }
-
- static void setupConf(Configuration conf) {
- // Up the handlers; this test needs more than usual.
- conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 15);
- // enable snapshot support
- conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
- // disable the ui
- conf.setInt("hbase.regionsever.info.port", -1);
- conf.setInt("hbase.master.info.port", -1);
- // change the flush size to a small amount, regulating number of store files
- conf.setInt("hbase.hregion.memstore.flush.size", 25000);
- // so make sure we get a compaction when doing a load, but keep around
- // some files in the store
- conf.setInt("hbase.hstore.compaction.min", 10);
- conf.setInt("hbase.hstore.compactionThreshold", 10);
- // block writes if we get to 12 store files
- conf.setInt("hbase.hstore.blockingStoreFiles", 12);
- conf.setInt("hbase.regionserver.msginterval", 100);
- conf.setBoolean("hbase.master.enabletable.roundrobin", true);
- // Avoid potentially aggressive splitting which would cause snapshot to fail
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
- ConstantSizeRegionSplitPolicy.class.getName());
- // Execute cleaner frequently to induce failures
- conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
- conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
- // Effectively disable TimeToLiveHFileCleaner. Don't want to fully disable it because that
- // will even trigger races between creating the directory containing back references and
- // the back reference itself.
- conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
- }
-
- @Before
- public void setup() throws Exception {
- createTable(TABLE_NAME, TEST_FAM);
- }
-
- @After
- public void tearDown() throws Exception {
- UTIL.deleteTable(TABLE_NAME);
- SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
- SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
- }
-
- @AfterClass
- public static void cleanupTest() throws Exception {
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- LOG.warn("failure shutting down cluster", e);
- }
- }
-
- /**
- * Verify that adding data to the cloned table will not affect the original, and vice-versa when
- * it is taken as an online snapshot.
- */
- @Ignore ("Flakey. Fix") @Test (timeout=300000)
- public void testOnlineSnapshotAppendIndependent() throws Exception {
- runTestSnapshotAppendIndependent(true);
- }
-
- /**
- * Verify that adding data to the cloned table will not affect the original, and vice-versa when
- * it is taken as an offline snapshot.
- */
- @Test (timeout=300000)
- @Ignore
- public void testOfflineSnapshotAppendIndependent() throws Exception {
- runTestSnapshotAppendIndependent(false);
- }
-
- /**
- * Verify that adding metadata to the cloned table will not affect the original, and vice-versa
- * when it is taken as an online snapshot.
- */
- @Test (timeout=300000)
- public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
- runTestSnapshotMetadataChangesIndependent(true);
- }
-
- /**
- * Verify that adding netadata to the cloned table will not affect the original, and vice-versa
- * when is taken as an online snapshot.
- */
- @Test (timeout=300000)
- @Ignore
- public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
- runTestSnapshotMetadataChangesIndependent(false);
- }
-
- /**
- * Verify that region operations, in this case splitting a region, are independent between the
- * cloned table and the original.
- */
- @Test (timeout=300000)
- @Ignore
- public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
- runTestRegionOperationsIndependent(false);
- }
-
- /**
- * Verify that region operations, in this case splitting a region, are independent between the
- * cloned table and the original.
- */
- @Test (timeout=300000)
- public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
- runTestRegionOperationsIndependent(true);
- }
-
- @Test (timeout=300000)
- @Ignore
- public void testOfflineSnapshotDeleteIndependent() throws Exception {
- runTestSnapshotDeleteIndependent(false);
- }
-
- @Ignore ("Flakey test") @Test (timeout=300000)
- public void testOnlineSnapshotDeleteIndependent() throws Exception {
- runTestSnapshotDeleteIndependent(true);
- }
-
- private static void waitOnSplit(Connection c, final Table t, int originalCount) throws Exception {
- for (int i = 0; i < 200; i++) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- try (RegionLocator locator = c.getRegionLocator(t.getName())) {
- if (locator.getAllRegionLocations().size() > originalCount) {
- return;
- }
- }
- }
- throw new Exception("Split did not increase the number of regions");
- }
-
- /*
- * Take a snapshot of a table, add data, and verify that this only
- * affects one table
- * @param online - Whether the table is online or not during the snapshot
- */
- private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-
- Admin admin = UTIL.getHBaseAdmin();
- final long startTime = System.currentTimeMillis();
- final TableName localTableName =
- TableName.valueOf(STRING_TABLE_NAME + startTime);
-
- try (Table original = createTable(localTableName, TEST_FAM)) {
- loadData(original, TEST_FAM);
- final int origTableRowCount = countRows(original);
-
- // Take a snapshot
- final String snapshotNameAsString = "snapshot_" + localTableName;
- byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
-
- SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
- snapshotNameAsString, rootDir, fs, online);
-
- if (!online) {
- tryDisable(admin, localTableName);
- }
- TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
- admin.cloneSnapshot(snapshotName, cloneTableName);
-
- try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
-
- // Make sure that all the regions are available before starting
- UTIL.waitUntilAllRegionsAssigned(cloneTableName);
-
- final int clonedTableRowCount = countRows(clonedTable);
-
- Assert.assertEquals(
- "The line counts of original and cloned tables do not match after clone. ",
- origTableRowCount, clonedTableRowCount);
-
- // Attempt to add data to the test
- final String rowKey = "new-row-" + System.currentTimeMillis();
-
- Put p = new Put(Bytes.toBytes(rowKey));
- p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
- original.put(p);
-
- // Verify that it is not present in the original table
- Assert.assertEquals("The row count of the original table was not modified by the put",
- origTableRowCount + 1, countRows(original));
- Assert.assertEquals(
- "The row count of the cloned table changed as a result of addition to the original",
- clonedTableRowCount, countRows(clonedTable));
-
- p = new Put(Bytes.toBytes(rowKey));
- p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
- clonedTable.put(p);
-
- // Verify that the new family is not in the restored table's description
- Assert.assertEquals(
- "The row count of the original table was modified by the put to the clone",
- origTableRowCount + 1, countRows(original));
- Assert.assertEquals("The row count of the cloned table was not modified by the put",
- clonedTableRowCount + 1, countRows(clonedTable));
- }
- }
- }
-
- /*
- * Take a snapshot of a table, do a split, and verify that this only affects one table
- * @param online - Whether the table is online or not during the snapshot
- */
- private void runTestRegionOperationsIndependent(boolean online) throws Exception {
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-
- // Create a table
- Admin admin = UTIL.getHBaseAdmin();
- final long startTime = System.currentTimeMillis();
- final TableName localTableName =
- TableName.valueOf(STRING_TABLE_NAME + startTime);
- Table original = createTable(localTableName, TEST_FAM);
- loadData(original, TEST_FAM);
- final int loadedTableCount = countRows(original);
- System.out.println("Original table has: " + loadedTableCount + " rows");
-
- final String snapshotNameAsString = "snapshot_" + localTableName;
-
- // Create a snapshot
- SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
- snapshotNameAsString, rootDir, fs, online);
-
- if (!online) {
- tryDisable(admin, localTableName);
- }
-
- TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
-
- // Clone the snapshot
- byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
- admin.cloneSnapshot(snapshotName, cloneTableName);
-
- // Verify that region information is the same pre-split
- ((ClusterConnection) UTIL.getConnection()).clearRegionCache();
- List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
-
- final int originalRegionCount = originalTableHRegions.size();
- final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
- Assert.assertEquals(
- "The number of regions in the cloned table is different than in the original table.",
- originalRegionCount, cloneTableRegionCount);
-
- // Split a region on the parent table
- admin.splitRegion(originalTableHRegions.get(0).getRegionName());
- waitOnSplit(UTIL.getConnection(), original, originalRegionCount);
-
- // Verify that the cloned table region is not split
- final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
- Assert.assertEquals(
- "The number of regions in the cloned table changed though none of its regions were split.",
- cloneTableRegionCount, cloneTableRegionCount2);
- }
-
- /*
- * Take a snapshot of a table, add metadata, and verify that this only
- * affects one table
- * @param online - Whether the table is online or not during the snapshot
- */
- private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-
- // Create a table
- Admin admin = UTIL.getHBaseAdmin();
- final long startTime = System.currentTimeMillis();
- final TableName localTableName =
- TableName.valueOf(STRING_TABLE_NAME + startTime);
- Table original = createTable(localTableName, TEST_FAM);
- loadData(original, TEST_FAM);
-
- final String snapshotNameAsString = "snapshot_" + localTableName;
-
- // Create a snapshot
- SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
- snapshotNameAsString, rootDir, fs, online);
-
- if (!online) {
- tryDisable(admin, localTableName);
- }
-
- TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
-
- // Clone the snapshot
- byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
- admin.cloneSnapshot(snapshotName, cloneTableName);
-
- // Add a new column family to the original table
- byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
- HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
-
- tryDisable(admin, localTableName);
- admin.addColumnFamily(localTableName, hcd);
-
- // Verify that it is not in the snapshot
- admin.enableTable(localTableName);
- UTIL.waitTableAvailable(localTableName);
-
- // get a description of the cloned table
- // get a list of its families
- // assert that the family is there
- HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
- HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
-
- Assert.assertTrue("The original family was not found. There is something wrong. ",
- originalTableDescriptor.hasFamily(TEST_FAM));
- Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
- clonedTableDescriptor.hasFamily(TEST_FAM));
-
- Assert.assertTrue("The new family was not found. ",
- originalTableDescriptor.hasFamily(TEST_FAM_2));
- Assert.assertTrue("The new family was not found. ",
- !clonedTableDescriptor.hasFamily(TEST_FAM_2));
- }
-
- private void tryDisable(Admin admin, TableName localTableName) throws IOException {
- int offlineRetry = 0;
- while ( offlineRetry < 5 && admin.isTableEnabled(localTableName)) {
- try {
- admin.disableTable(localTableName);
- } catch (IOException ioe) {
- LOG.warn("Error disabling the table", ioe);
- }
- offlineRetry ++;
- }
- }
-
- /*
- * Take a snapshot of a table, add data, and verify that deleting the snapshot does not affect
- * either table.
- * @param online - Whether the table is online or not during the snapshot
- */
- private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-
- final Admin admin = UTIL.getHBaseAdmin();
- final long startTime = System.currentTimeMillis();
- final TableName localTableName =
- TableName.valueOf(STRING_TABLE_NAME + startTime);
-
- try (Table original = createTable(localTableName, TEST_FAM)) {
- loadData(original, TEST_FAM);
- }
-
- // Take a snapshot
- final String snapshotNameAsString = "snapshot_" + localTableName;
- byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
-
- SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
- snapshotNameAsString, rootDir, fs, online);
-
- if (!online) {
- tryDisable(admin, localTableName);
- }
-
- TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
- admin.cloneSnapshot(snapshotName, cloneTableName);
-
- UTIL.waitUntilAllRegionsAssigned(cloneTableName);
-
- // Ensure the original table does not reference the HFiles anymore
- admin.majorCompact(localTableName);
-
- // Deleting the snapshot used to break the cloned table by deleting in-use HFiles
- admin.deleteSnapshot(snapshotName);
-
- // Wait for cleaner run and DFS heartbeats so that anything that is deletable is fully deleted
- do {
- Thread.sleep(5000);
- } while (!admin.listSnapshots(snapshotNameAsString).isEmpty());
-
- try (Table original = UTIL.getConnection().getTable(localTableName)) {
- try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
- // Verify that all regions of both tables are readable
- final int origTableRowCount = countRows(original);
- final int clonedTableRowCount = countRows(clonedTable);
- Assert.assertEquals(origTableRowCount, clonedTableRowCount);
- }
- }
- }
-
- protected Table createTable(final TableName table, byte[] family) throws Exception {
- Table t = UTIL.createTable(table, family);
- // Wait for everything to be ready with the table
- UTIL.waitUntilAllRegionsAssigned(table);
-
- // At this point the table should be good to go.
- return t;
- }
-
- protected void loadData(final Table table, byte[]... families) throws Exception {
- UTIL.loadTable(table, families);
- }
-
- protected int countRows(final Table table, final byte[]... families) throws Exception {
- return UTIL.countRows(table, families);
- }
-}
[07/14] hbase git commit: HBASE-14676 HBaseTestCase clean out: Purge
Incommon Interface and Table and Region implementations: ADDENDUM -- forgot
to add file
Posted by sy...@apache.org.
HBASE-14676 HBaseTestCase clean out: Purge Incommon Interface and Table and Region implementations: ADDENDUM -- forgot to add file
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80a6fd01
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80a6fd01
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80a6fd01
Branch: refs/heads/hbase-12439
Commit: 80a6fd01a170fd2e4798eaa44f6cd3e0c7b63d4c
Parents: 9925e9a
Author: stack <st...@apache.org>
Authored: Sat Oct 24 11:09:08 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 11:09:18 2015 -0700
----------------------------------------------------------------------
.../hbase/regionserver/RegionAsTable.java | 324 +++++++++++++++++++
1 file changed, 324 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/80a6fd01/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/RegionAsTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/RegionAsTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/RegionAsTable.java
new file mode 100644
index 0000000..f65bc5d
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/RegionAsTable.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+
+/**
+ * An implementation of {@link Table} that sits directly on a Region; it decorates the passed in
+ * Region instance with the Table API. Some API is not implemented yet (throws
+ * {@link UnsupportedOperationException}) mostly because no need as yet or it necessitates copying
+ * a load of code local from RegionServer.
+ *
+ * <p>Use as an instance of a {@link Table} in-the-small -- no networking or servers
+ * necessary -- or to write a test that can run directly against the datastore and then
+ * over the network.
+ */
+public class RegionAsTable implements Table {
+ private final Region region;
+
+ /**
+ * @param region Region to decorate with Table API.
+ */
+ public RegionAsTable(final Region region) {
+ this.region = region;
+ }
+
+ @Override
+ public TableName getName() {
+ return this.region.getTableDesc().getTableName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return this.region.getTableDesc();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ if (!get.isCheckExistenceOnly()) throw new IllegalArgumentException();
+ return get(get) != null;
+ }
+
+ @Override
+ public boolean[] existsAll(List<Get> gets) throws IOException {
+ boolean [] results = new boolean[gets.size()];
+ int index = 0;
+ for (Get get: gets) {
+ results[index++] = exists(get);
+ }
+ return results;
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results,
+ Callback<R> callback)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ return this.region.get(get);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ Result [] results = new Result[gets.size()];
+ int index = 0;
+ for (Get get: gets) {
+ results[index++] = get(get);
+ }
+ return results;
+ }
+
+ static class RegionScannerToResultScannerAdaptor implements ResultScanner {
+ private static final Result [] EMPTY_RESULT_ARRAY = new Result[0];
+ private final RegionScanner regionScanner;
+
+ RegionScannerToResultScannerAdaptor(final RegionScanner regionScanner) {
+ this.regionScanner = regionScanner;
+ }
+
+ @Override
+ public Iterator<Result> iterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Result next() throws IOException {
+ List<Cell> cells = new ArrayList<Cell>();
+ return regionScanner.next(cells)? Result.create(cells): null;
+ }
+
+ @Override
+ public Result[] next(int nbRows) throws IOException {
+ List<Result> results = new ArrayList<Result>(nbRows);
+ for (int i = 0; i < nbRows; i++) {
+ Result result = next();
+ if (result == null) break;
+ results.add(result);
+ }
+ return results.toArray(EMPTY_RESULT_ARRAY);
+ }
+
+ @Override
+ public void close() {
+ try {
+ regionScanner.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean renewLease() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ return new RegionScannerToResultScannerAdaptor(this.region.getScanner(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ return getScanner(new Scan().addFamily(family));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ return getScanner(new Scan().addColumn(family, qualifier));
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ this.region.put(put);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ for (Put put: puts) put(put);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
+ byte[] value, Put put)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ this.region.delete(delete);
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ for(Delete delete: deletes) delete(delete);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value,
+ Delete delete)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
+ CompareOp compareOp, byte[] value, Delete delete)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ return this.region.append(append, HConstants.NO_NONCE, HConstants.NO_NONCE);
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ return this.region.increment(increment, HConstants.NO_NONCE, HConstants.NO_NONCE);
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount,
+ Durability durability)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * This call will NOT close the underlying region.
+ */
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey,
+ byte[] endKey, Call<T, R> callable)
+ throws ServiceException, Throwable {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey,
+ byte[] endKey, Call<T, R> callable, Callback<R> callback)
+ throws ServiceException, Throwable {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(MethodDescriptor
+ methodDescriptor, Message request,
+ byte[] startKey, byte[] endKey, R responsePrototype)
+ throws ServiceException, Throwable {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
+ throws ServiceException, Throwable {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
+ byte[] value, RowMutations mutation)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
[02/14] hbase git commit: HBASE-14670 Remove deprecated HBaseTestCase
from TestBlocksRead
Posted by sy...@apache.org.
HBASE-14670 Remove deprecated HBaseTestCase from TestBlocksRead
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/418c4422
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/418c4422
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/418c4422
Branch: refs/heads/hbase-12439
Commit: 418c4422f7fc99ce58bd10c20090f34211da3b13
Parents: 6875e6d
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Sat Oct 24 05:35:00 2015 -0700
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Sat Oct 24 05:35:00 2015 -0700
----------------------------------------------------------------------
.../hbase/regionserver/TestBlocksRead.java | 58 ++++++++------------
1 file changed, 22 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/418c4422/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
index c4b26b9..900d4ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
@@ -26,11 +26,10 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -48,44 +47,35 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.*;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
@Category({RegionServerTests.class, MediumTests.class})
-public class TestBlocksRead extends HBaseTestCase {
+public class TestBlocksRead {
private static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
+ @Rule public TestName testName = new TestName();
+
static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL,
BloomType.ROW, BloomType.NONE };
private static BlockCache blockCache;
-
- private HBaseConfiguration getConf() {
- HBaseConfiguration conf = new HBaseConfiguration();
-
- // disable compactions in this test.
- conf.setInt("hbase.hstore.compactionThreshold", 10000);
- return conf;
- }
-
Region region = null;
- private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString();
+ private Configuration conf = TEST_UTIL.getConfiguration();
- /**
- * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
- */
- @SuppressWarnings("deprecation")
- @Before
- protected void setUp() throws Exception {
- super.setUp();
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // disable compactions in this test.
+ TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10000);
}
- @SuppressWarnings("deprecation")
- @After
- protected void tearDown() throws Exception {
- super.tearDown();
+ @AfterClass
+ public static void tearDown() throws Exception {
EnvironmentEdgeManagerTestHelper.reset();
}
@@ -99,7 +89,7 @@ public class TestBlocksRead extends HBaseTestCase {
* @return created and initialized region.
*/
private Region initHRegion(byte[] tableName, String callingMethod,
- HBaseConfiguration conf, String family) throws IOException {
+ Configuration conf, String family) throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor familyDesc;
for (int i = 0; i < BLOOM_TYPE.length; i++) {
@@ -223,8 +213,7 @@ public class TestBlocksRead extends HBaseTestCase {
byte[] TABLE = Bytes.toBytes("testBlocksRead");
String FAMILY = "cf1";
Cell kvs[];
- HBaseConfiguration conf = getConf();
- this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
try {
putData(FAMILY, "row", "col1", 1);
@@ -280,8 +269,7 @@ public class TestBlocksRead extends HBaseTestCase {
byte[] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
String FAMILY = "cf1";
Cell kvs[];
- HBaseConfiguration conf = getConf();
- this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
try {
// File 1
@@ -388,8 +376,7 @@ public class TestBlocksRead extends HBaseTestCase {
byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
String FAMILY = "cf1";
- HBaseConfiguration conf = getConf();
- this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
try {
putData(FAMILY, "row", "col1", 1);
@@ -433,8 +420,7 @@ public class TestBlocksRead extends HBaseTestCase {
byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
String FAMILY = "cf1";
Cell kvs[];
- HBaseConfiguration conf = getConf();
- this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
try {
deleteFamily(FAMILY, "row", 200);
for (int i = 0; i < 100; i++) {
[10/14] hbase git commit: HBASE-14678 Experiment: Temporarily disable
balancer and a few others to see if root of crashed/timedout JVMs;
ADD TestDistributedLogSplitting to the mix;
ADDENDUM added TestMobSnapshotCloneIndependence to removed set of dod
Posted by sy...@apache.org.
HBASE-14678 Experiment: Temporarily disable balancer and a few others to see if root of crashed/timedout JVMs; ADD TestDistributedLogSplitting to the mix; ADDENDUM added TestMobSnapshotCloneIndependence to removed set of dodgy tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8528f66
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8528f66
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8528f66
Branch: refs/heads/hbase-12439
Commit: f8528f66ec253fbb3c08d2c3352dd73d9832a43b
Parents: 996866d
Author: stack <st...@apache.org>
Authored: Sat Oct 24 17:42:16 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 17:42:16 2015 -0700
----------------------------------------------------------------------
.../TestMobSnapshotCloneIndependence.java | 69 --------------------
1 file changed, 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8528f66/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
deleted file mode 100644
index dcf20e5..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.client;
-
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test to verify that the cloned table is independent of the table from which it was cloned
- */
-@Category(LargeTests.class)
-public class TestMobSnapshotCloneIndependence extends TestSnapshotCloneIndependence {
- private static final Log LOG = LogFactory.getLog(TestMobSnapshotCloneIndependence.class);
-
- /**
- * Setup the config for the cluster and start it
- * @throws Exception on failure
- */
- @BeforeClass
- public static void setupCluster() throws Exception {
- setupConf(UTIL.getConfiguration());
- UTIL.startMiniCluster(NUM_RS);
- }
-
- protected static void setupConf(Configuration conf) {
- TestSnapshotCloneIndependence.setupConf(conf);
- conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
- }
-
- @Override
- protected Table createTable(final TableName table, byte[] family) throws Exception {
- return MobSnapshotTestingUtils.createMobTable(UTIL, table, family);
- }
-
- @Override
- public void loadData(final Table table, byte[]... families) throws Exception {
- SnapshotTestingUtils.loadData(UTIL, table.getName(), 1000, families);
- }
-
- @Override
- protected int countRows(final Table table, final byte[]... families) throws Exception {
- return MobSnapshotTestingUtils.countMobRows(table, families);
- }
-}
\ No newline at end of file
[04/14] hbase git commit: HBASE-14683 Fix Batching in buffered
mutator is awful when adding lists of mutations.
Posted by sy...@apache.org.
HBASE-14683 Fix Batching in buffered mutator is awful when adding lists of mutations.
Summary: Send the list of mutations to AsyncProcess only after done adding the list otherwise there's a lot of contention
Test Plan: UnitTests.
Differential Revision: https://reviews.facebook.net/D49251
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/caae3b2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/caae3b2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/caae3b2e
Branch: refs/heads/hbase-12439
Commit: caae3b2e5fe889d57dc69964e76faffdc16ff7ae
Parents: f9efeaa
Author: Elliott Clark <ec...@apache.org>
Authored: Thu Oct 22 20:24:48 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Sat Oct 24 10:05:39 2015 -0700
----------------------------------------------------------------------
.../hbase/client/BufferedMutatorImpl.java | 46 ++++++++------------
.../master/cleaner/TestSnapshotFromMaster.java | 15 ++++---
2 files changed, 25 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/caae3b2e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
index 0b222b1..5341d47 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
@@ -27,6 +27,7 @@ import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -98,48 +99,35 @@ public class BufferedMutatorImpl implements BufferedMutator {
@Override
public synchronized void mutate(Mutation m) throws InterruptedIOException,
RetriesExhaustedWithDetailsException {
- doMutate(m);
+ mutate(Arrays.asList(m));
}
@Override
public synchronized void mutate(List<? extends Mutation> ms) throws InterruptedIOException,
RetriesExhaustedWithDetailsException {
- for (Mutation m : ms) {
- doMutate(m);
- }
- }
-
- /**
- * Add the put to the buffer. If the buffer is already too large, sends the buffer to the
- * cluster.
- *
- * @throws RetriesExhaustedWithDetailsException if there is an error on the cluster.
- * @throws InterruptedIOException if we were interrupted.
- */
- private void doMutate(Mutation m) throws InterruptedIOException,
- RetriesExhaustedWithDetailsException {
if (closed) {
throw new IllegalStateException("Cannot put when the BufferedMutator is closed.");
}
- if (!(m instanceof Put) && !(m instanceof Delete)) {
- throw new IllegalArgumentException("Pass a Delete or a Put");
- }
- // This behavior is highly non-intuitive... it does not protect us against
- // 94-incompatible behavior, which is a timing issue because hasError, the below code
- // and setter of hasError are not synchronized. Perhaps it should be removed.
- if (ap.hasError()) {
- writeAsyncBuffer.add(m);
- backgroundFlushCommits(true);
+ for (Mutation m : ms) {
+ if (m instanceof Put) {
+ validatePut((Put) m);
+ }
+ currentWriteBufferSize += m.heapSize();
}
- if (m instanceof Put) {
- validatePut((Put) m);
- }
+ // This behavior is highly non-intuitive... it does not protect us against
+ // 94-incompatible behavior, which is a timing issue because hasError, the below code
+ // and setter of hasError are not synchronized. Perhaps it should be removed.
+ if (ap.hasError()) {
+ writeAsyncBuffer.addAll(ms);
+ backgroundFlushCommits(true);
+ } else {
+ writeAsyncBuffer.addAll(ms);
+ }
- currentWriteBufferSize += m.heapSize();
- writeAsyncBuffer.add(m);
+ // Now try and queue what needs to be queued.
while (currentWriteBufferSize > writeBufferSize) {
backgroundFlushCommits(false);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/caae3b2e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index e00134a..687b6ae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -111,7 +111,7 @@ public class TestSnapshotFromMaster {
conf.setInt("hbase.hregion.memstore.flush.size", 25000);
// so make sure we get a compaction when doing a load, but keep around some
// files in the store
- conf.setInt("hbase.hstore.compaction.min", 3);
+ conf.setInt("hbase.hstore.compaction.min", 2);
conf.setInt("hbase.hstore.compactionThreshold", 5);
// block writes if we get to 12 store files
conf.setInt("hbase.hstore.blockingStoreFiles", 12);
@@ -288,11 +288,12 @@ public class TestSnapshotFromMaster {
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
htd.setCompactionEnabled(false);
UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
- // load the table (creates 4 hfiles)
- UTIL.loadTable(UTIL.getConnection().getTable(TABLE_NAME), TEST_FAM);
- UTIL.flush(TABLE_NAME);
- // Put some more data into the table so for sure we get more storefiles.
- UTIL.loadTable(UTIL.getConnection().getTable(TABLE_NAME), TEST_FAM);
+
+ // load the table (creates at least 4 hfiles)
+ for ( int i = 0; i < 5; i++) {
+ UTIL.loadTable(UTIL.getConnection().getTable(TABLE_NAME), TEST_FAM);
+ UTIL.flush(TABLE_NAME);
+ }
// disable the table so we can take a snapshot
admin.disableTable(TABLE_NAME);
@@ -319,7 +320,7 @@ public class TestSnapshotFromMaster {
List<HRegion> regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
for (HRegion region : regions) {
region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it.
- region.compactStores(); // min is 3 so will compact and archive
+ region.compactStores(); // min is 2 so will compact and archive
}
LOG.info("After compaction File-System state");
FSUtils.logFileSystemState(fs, rootDir, LOG);
[03/14] hbase git commit: Revert "HBASE-14684 Try to remove all
MiniMapReduceCluster in unit tests (Heng Chen)"
Posted by sy...@apache.org.
Revert "HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests (Heng Chen)"
This reverts commit ef7f8467af52cd1d6563ad5b1f6624be6872a165.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f9efeaad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f9efeaad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f9efeaad
Branch: refs/heads/hbase-12439
Commit: f9efeaad1d5396e7cfca461ddeaa5931dc473f5f
Parents: 418c442
Author: stack <st...@apache.org>
Authored: Sat Oct 24 10:00:27 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 10:00:27 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapred/TestTableInputFormat.java | 8 ++++----
.../apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java | 2 ++
.../hbase/mapreduce/MultiTableInputFormatTestBase.java | 3 +++
.../hbase/mapreduce/TableSnapshotInputFormatTestBase.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestCellCounter.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestCopyTable.java | 2 ++
.../apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 6 ++++++
.../hadoop/hbase/mapreduce/TestHFileOutputFormat2.java | 6 ++++++
.../hadoop/hbase/mapreduce/TestHRegionPartitioner.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestHashTable.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestImportExport.java | 2 ++
.../mapreduce/TestImportTSVWithOperationAttributes.java | 2 ++
.../apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java | 2 ++
.../hbase/mapreduce/TestImportTSVWithVisibilityLabels.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestImportTsv.java | 2 ++
.../hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestRowCounter.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestSyncTable.java | 2 ++
.../apache/hadoop/hbase/mapreduce/TestTableInputFormat.java | 2 ++
.../hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java | 3 +++
.../hadoop/hbase/mapreduce/TestTableMapReduceBase.java | 2 ++
.../apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java | 2 ++
.../org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java | 2 ++
.../org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java | 2 ++
.../apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java | 1 +
.../hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java | 1 +
.../hadoop/hbase/snapshot/TestSecureExportSnapshot.java | 1 +
27 files changed, 63 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
index 1975c59..8498341 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
@@ -34,7 +34,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Connection;
@@ -81,7 +80,7 @@ public class TestTableInputFormat {
private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
+ private static MiniMRCluster mrCluster;
static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] columns = new byte[][] { FAMILY };
@@ -89,10 +88,12 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
+ mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
@@ -343,8 +344,7 @@ public class TestTableInputFormat {
}
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
- Configuration conf = UTIL.getConfiguration();
- final JobConf job = new JobConf(conf);
+ final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
job.setInputFormat(clazz);
job.setOutputFormat(NullOutputFormat.class);
job.setMapperClass(ExampleVerifier.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
index daa6e71..628bb96 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
@@ -90,10 +90,12 @@ public class TestTableMapReduceUtil {
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
+ UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
index 3cd8b4d..38e97b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
@@ -83,10 +83,13 @@ public abstract class MultiTableInputFormatTestBase {
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
}
}
+ // start MR cluster
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 58db63e..d6c5d32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -154,12 +154,14 @@ public abstract class TableSnapshotInputFormatTestBase {
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
+ util.startMiniMapReduceCluster();
try {
Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
TableName tableName = TableName.valueOf("testWithMapReduce");
testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
expectedNumSplits, shutdownCluster);
} finally {
+ util.shutdownMiniMapReduceCluster();
tearDownCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
index 54a81b7..78707fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
@@ -63,12 +63,14 @@ public class TestCellCounter {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
+ UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
FileUtil.fullyDelete(new File(OUTPUT_DIR));
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
index 626383b..1c13157 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -62,10 +62,12 @@ public class TestCopyTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 25642fe..90b92bb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -403,6 +403,7 @@ public class TestHFileOutputFormat {
assertEquals("Should make 5 regions", numRegions, 5);
// Generate the bulk load files
+ util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table",
@@ -472,6 +473,7 @@ public class TestHFileOutputFormat {
assertEquals("Data should remain after reopening of regions",
tableDigestBefore, util.checksumRows(table));
} finally {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -935,6 +937,7 @@ public class TestHFileOutputFormat {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
+ util.startMiniMapReduceCluster();
try (RegionLocator regionLocator = util.getConnection().getRegionLocator(TABLE_NAME)) {
for (int i = 0; i < 2; i++) {
@@ -975,6 +978,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -1015,6 +1019,7 @@ public class TestHFileOutputFormat {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
+ util.startMiniMapReduceCluster();
try (RegionLocator locator = util.getConnection().getRegionLocator(TABLE_NAME)) {
runIncrementalPELoad(conf, table, locator, testDir);
@@ -1053,6 +1058,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index 69b3276..7a13e5d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -430,6 +430,7 @@ public class TestHFileOutputFormat2 {
assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
// Generate the bulk load files
+ util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table.getTableDescriptor(), r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@@ -510,6 +511,7 @@ public class TestHFileOutputFormat2 {
} finally {
testDir.getFileSystem(conf).delete(testDir, true);
util.deleteTable(TABLE_NAME);
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -974,6 +976,7 @@ public class TestHFileOutputFormat2 {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
+ util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@@ -1015,6 +1018,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -1056,6 +1060,7 @@ public class TestHFileOutputFormat2 {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
+ util.startMiniMapReduceCluster();
RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@@ -1095,6 +1100,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
index 9b461d7..33d0e74 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
@@ -36,10 +36,12 @@ public class TestHRegionPartitioner {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
+ UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
index 34020b5..2d03ac1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
@@ -57,10 +57,12 @@ public class TestHashTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index b9ad6af..a3fc4f6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -111,12 +111,14 @@ public class TestImportExport {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.startMiniCluster();
+ UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR =
new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
index 66cf0ff..7360a63 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
@@ -99,10 +99,12 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
util.startMiniCluster();
+ util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
index 3fc129f..445620f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
@@ -90,10 +90,12 @@ public class TestImportTSVWithTTLs implements Configurable {
conf.setInt("hfile.format.version", 3);
conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
util.startMiniCluster();
+ util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
index 2b96d8c..3aec669 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
@@ -124,6 +124,7 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
// Wait for the labels table to become available
util.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
createLabels();
+ util.startMiniMapReduceCluster();
}
private static void createLabels() throws IOException, InterruptedException {
@@ -147,6 +148,7 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
@AfterClass
public static void releaseCluster() throws Exception {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 9b7c4ae..66fdf93 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -104,10 +104,12 @@ public class TestImportTsv implements Configurable {
@BeforeClass
public static void provisionCluster() throws Exception {
util.startMiniCluster();
+ util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
+ util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
index 03052fb..9e1d2bf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
@@ -76,11 +76,13 @@ public class TestMultithreadedTableMapper {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
+ UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
index fe78844..b0bf558 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
@@ -64,6 +64,7 @@ public class TestRowCounter {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
+ TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
@@ -75,6 +76,7 @@ public class TestRowCounter {
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
+ TEST_UTIL.shutdownMiniMapReduceCluster();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
index aca1705..841d8cb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
@@ -60,10 +60,12 @@ public class TestSyncTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
index b0a4243..0ec46e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
@@ -84,10 +84,12 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
+ mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
index 46e1a0e..23714b7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
@@ -84,10 +84,13 @@ public abstract class TestTableInputFormatScanBase {
// create and fill table
table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
+ // start MR cluster
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
index 77ebb48..5436a1d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
@@ -79,10 +79,12 @@ public abstract class TestTableMapReduceBase {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
+ UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
index 7e75830..03da1ed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
@@ -168,6 +168,7 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
+ UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");
@@ -184,6 +185,7 @@ public class TestTimeRangeMapRed {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
+ UTIL.shutdownMiniMapReduceCluster();
if (job != null) {
FileUtil.fullyDelete(
new File(job.getConfiguration().get("hadoop.tmp.dir")));
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index 60b050f..a12887e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -73,10 +73,12 @@ public class TestWALPlayer {
@BeforeClass
public static void beforeClass() throws Exception {
cluster = TEST_UTIL.startMiniCluster();
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index a7ac1bb..9b85ac7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -87,10 +87,12 @@ public class TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
index 64c5bc0..b6cb339 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
@@ -45,6 +45,7 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
index 403e8bb..4a89db8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
@@ -45,6 +45,7 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f9efeaad/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
index ea9a584..19d5965 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
@@ -46,6 +46,7 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
[09/14] hbase git commit: HBASE-14266 RegionServers have a lock
contention of Configuration.getProps
Posted by sy...@apache.org.
HBASE-14266 RegionServers have a lock contention of Configuration.getProps
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/996866d3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/996866d3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/996866d3
Branch: refs/heads/hbase-12439
Commit: 996866d3ae66b28aa6d25b69db7646f9847e9b21
Parents: df36aef
Author: stack <st...@apache.org>
Authored: Sat Oct 24 14:43:53 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 14:43:53 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/HBaseConfiguration.java | 3 +-
.../example/ZooKeeperScanPolicyObserver.java | 2 +-
.../compactions/PartitionedMobCompactor.java | 2 +-
.../hadoop/hbase/regionserver/HStore.java | 2 +-
.../hadoop/hbase/regionserver/ScanInfo.java | 67 +++++-
.../hadoop/hbase/regionserver/StoreScanner.java | 226 +++++++++----------
.../TestPartitionedMobCompactor.java | 2 +-
.../regionserver/NoOpScanPolicyObserver.java | 4 +-
.../hbase/regionserver/TestCompaction.java | 2 +-
.../TestDefaultCompactSelection.java | 2 +-
.../hbase/regionserver/TestDefaultMemStore.java | 8 +-
.../hbase/regionserver/TestMajorCompaction.java | 2 +-
.../regionserver/TestMobStoreCompaction.java | 3 +-
.../hbase/regionserver/TestQueryMatcher.java | 16 +-
.../regionserver/TestReversibleScanners.java | 2 +-
.../hbase/regionserver/TestStoreScanner.java | 11 +-
.../hbase/regionserver/TestStripeCompactor.java | 2 +-
.../hbase/util/TestCoprocessorScanPolicy.java | 12 +-
18 files changed, 201 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 1be66a1..5a6a168 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -36,11 +36,10 @@ import org.apache.hadoop.hbase.util.VersionInfo;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class HBaseConfiguration extends Configuration {
-
private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
/**
- * Instantinating HBaseConfiguration() is deprecated. Please use
+ * Instantiating HBaseConfiguration() is deprecated. Please use
* HBaseConfiguration#create() to construct a plain Configuration
* @deprecated Please use create() instead.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
index a65b27d..420799f 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
@@ -188,7 +188,7 @@ public class ZooKeeperScanPolicyObserver extends BaseRegionObserver {
}
long ttl = Math.max(EnvironmentEdgeManager.currentTime() -
Bytes.toLong(data), oldSI.getTtl());
- return new ScanInfo(store.getFamily(), ttl,
+ return new ScanInfo(oldSI.getConfiguration(), store.getFamily(), ttl,
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
index e0ae481..dabedfd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
@@ -551,7 +551,7 @@ public class PartitionedMobCompactor extends MobCompactor {
Scan scan = new Scan();
scan.setMaxVersions(column.getMaxVersions());
long ttl = HStore.determineTTLFromFamily(column);
- ScanInfo scanInfo = new ScanInfo(column, ttl, 0, CellComparator.COMPARATOR);
+ ScanInfo scanInfo = new ScanInfo(conf, column, ttl, 0, CellComparator.COMPARATOR);
StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners, 0L,
HConstants.LATEST_TIMESTAMP);
return scanner;
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 3c9e00a..c0f1d55 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -232,7 +232,7 @@ public class HStore implements Store {
long ttl = determineTTLFromFamily(family);
// Why not just pass a HColumnDescriptor in here altogether? Even if have
// to clone it?
- scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
+ scanInfo = new ScanInfo(conf, family, ttl, timeToPurgeDeletes, this.comparator);
String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName());
this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
Configuration.class, CellComparator.class }, new Object[] { conf, this.comparator });
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
index 83a1bdd..42f7369 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
@@ -19,15 +19,22 @@
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Immutable information for scans over a store.
*/
+// Has to be public for PartitionedMobCompactor to access; ditto on tests making use of a few of
+// the accessors below. Shutdown access. TODO
+@VisibleForTesting
@InterfaceAudience.Private
public class ScanInfo {
private byte[] family;
@@ -37,25 +44,32 @@ public class ScanInfo {
private KeepDeletedCells keepDeletedCells;
private long timeToPurgeDeletes;
private CellComparator comparator;
+ private long tableMaxRowSize;
+ private boolean usePread;
+ private long cellsPerTimeoutCheck;
+ private boolean parallelSeekEnabled;
+ private final Configuration conf;
public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
+ (2 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_INT)
- + (2 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN);
+ + (4 * Bytes.SIZEOF_LONG) + (3 * Bytes.SIZEOF_BOOLEAN));
/**
+ * @param conf
* @param family {@link HColumnDescriptor} describing the column family
* @param ttl Store's TTL (in ms)
* @param timeToPurgeDeletes duration in ms after which a delete marker can
* be purged during a major compaction.
* @param comparator The store's comparator
*/
- public ScanInfo(final HColumnDescriptor family, final long ttl, final long timeToPurgeDeletes,
- final CellComparator comparator) {
- this(family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family
+ public ScanInfo(final Configuration conf, final HColumnDescriptor family, final long ttl,
+ final long timeToPurgeDeletes, final CellComparator comparator) {
+ this(conf, family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family
.getKeepDeletedCells(), timeToPurgeDeletes, comparator);
}
/**
+ * @param conf
* @param family Name of this store's column family
* @param minVersions Store's MIN_VERSIONS setting
* @param maxVersions Store's VERSIONS setting
@@ -65,9 +79,9 @@ public class ScanInfo {
* @param keepDeletedCells Store's keepDeletedCells setting
* @param comparator The store's comparator
*/
- public ScanInfo(final byte[] family, final int minVersions, final int maxVersions,
- final long ttl, final KeepDeletedCells keepDeletedCells, final long timeToPurgeDeletes,
- final CellComparator comparator) {
+ public ScanInfo(final Configuration conf, final byte[] family, final int minVersions,
+ final int maxVersions, final long ttl, final KeepDeletedCells keepDeletedCells,
+ final long timeToPurgeDeletes, final CellComparator comparator) {
this.family = family;
this.minVersions = minVersions;
this.maxVersions = maxVersions;
@@ -75,13 +89,44 @@ public class ScanInfo {
this.keepDeletedCells = keepDeletedCells;
this.timeToPurgeDeletes = timeToPurgeDeletes;
this.comparator = comparator;
+ this.tableMaxRowSize =
+ conf.getLong(HConstants.TABLE_MAX_ROWSIZE_KEY, HConstants.TABLE_MAX_ROWSIZE_DEFAULT);
+ this.usePread = conf.getBoolean("hbase.storescanner.use.pread", false);
+ long perHeartbeat =
+ conf.getLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK,
+ StoreScanner.DEFAULT_HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK);
+ this.cellsPerTimeoutCheck = perHeartbeat > 0?
+ perHeartbeat: StoreScanner.DEFAULT_HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK;
+ this.parallelSeekEnabled =
+ conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false);
+ this.conf = conf;
+ }
+
+ public Configuration getConfiguration() {
+ return this.conf;
+ }
+
+ long getTableMaxRowSize() {
+ return this.tableMaxRowSize;
+ }
+
+ boolean isUsePread() {
+ return this.usePread;
+ }
+
+ long getCellsPerTimeoutCheck() {
+ return this.cellsPerTimeoutCheck;
+ }
+
+ boolean isParallelSeekEnabled() {
+ return this.parallelSeekEnabled;
}
- public byte[] getFamily() {
+ byte[] getFamily() {
return family;
}
- public int getMinVersions() {
+ int getMinVersions() {
return minVersions;
}
@@ -93,7 +138,7 @@ public class ScanInfo {
return ttl;
}
- public KeepDeletedCells getKeepDeletedCells() {
+ KeepDeletedCells getKeepDeletedCells() {
return keepDeletedCells;
}
@@ -104,4 +149,4 @@ public class ScanInfo {
public CellComparator getComparator() {
return comparator;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 26fc338..0377ec2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -31,7 +31,6 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
@@ -49,6 +48,8 @@ import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState;
import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Scanner scans both the memstore and the Store. Coalesce KeyValue stream
* into List<KeyValue> for a single row.
@@ -57,7 +58,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
public class StoreScanner extends NonReversedNonLazyKeyValueScanner
implements KeyValueScanner, InternalScanner, ChangedReadersObserver {
private static final Log LOG = LogFactory.getLog(StoreScanner.class);
- protected Store store;
+ // In unit tests, the store could be null
+ protected final Store store;
protected ScanQueryMatcher matcher;
protected KeyValueHeap heap;
protected boolean cacheBlocks;
@@ -69,13 +71,13 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
// Used to indicate that the scanner has closed (see HBASE-1107)
// Doesnt need to be volatile because it's always accessed via synchronized methods
protected boolean closing = false;
- protected final boolean isGet;
+ protected final boolean get;
protected final boolean explicitColumnQuery;
protected final boolean useRowColBloom;
/**
* A flag that enables StoreFileScanner parallel-seeking
*/
- protected boolean isParallelSeekEnabled = false;
+ protected boolean parallelSeekEnabled = false;
protected ExecutorService executor;
protected final Scan scan;
protected final NavigableSet<byte[]> columns;
@@ -133,58 +135,39 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
AFTER_SEEK,
COMPACT_COMPLETE
}
-
+
/** An internal constructor. */
- protected StoreScanner(Store store, boolean cacheBlocks, Scan scan,
- final NavigableSet<byte[]> columns, long ttl, int minVersions, long readPt) {
+ protected StoreScanner(Store store, Scan scan, final ScanInfo scanInfo,
+ final NavigableSet<byte[]> columns, long readPt, boolean cacheBlocks) {
this.readPt = readPt;
this.store = store;
this.cacheBlocks = cacheBlocks;
- isGet = scan.isGetScan();
+ get = scan.isGetScan();
int numCol = columns == null ? 0 : columns.size();
explicitColumnQuery = numCol > 0;
this.scan = scan;
this.columns = columns;
this.now = EnvironmentEdgeManager.currentTime();
- this.oldestUnexpiredTS = now - ttl;
- this.minVersions = minVersions;
-
- if (store != null && ((HStore)store).getHRegion() != null
- && ((HStore)store).getHRegion().getBaseConf() != null) {
- Configuration conf = ((HStore) store).getHRegion().getBaseConf();
- this.maxRowSize =
- conf.getLong(HConstants.TABLE_MAX_ROWSIZE_KEY, HConstants.TABLE_MAX_ROWSIZE_DEFAULT);
- this.scanUsePread = conf.getBoolean("hbase.storescanner.use.pread", scan.isSmall());
-
- long tmpCellsPerTimeoutCheck =
- conf.getLong(HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK,
- DEFAULT_HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK);
- this.cellsPerHeartbeatCheck =
- tmpCellsPerTimeoutCheck > 0 ? tmpCellsPerTimeoutCheck
- : DEFAULT_HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK;
- } else {
- this.maxRowSize = HConstants.TABLE_MAX_ROWSIZE_DEFAULT;
- this.scanUsePread = scan.isSmall();
- this.cellsPerHeartbeatCheck = DEFAULT_HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK;
- }
-
- // We look up row-column Bloom filters for multi-column queries as part of
- // the seek operation. However, we also look the row-column Bloom filter
- // for multi-row (non-"get") scans because this is not done in
- // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
- useRowColBloom = numCol > 1 || (!isGet && numCol == 1);
-
- // The parallel-seeking is on :
- // 1) the config value is *true*
- // 2) store has more than one store file
- if (store != null && ((HStore)store).getHRegion() != null
- && store.getStorefilesCount() > 1) {
- RegionServerServices rsService = ((HStore)store).getHRegion().getRegionServerServices();
- if (rsService == null || !rsService.getConfiguration().getBoolean(
- STORESCANNER_PARALLEL_SEEK_ENABLE, false)) return;
- isParallelSeekEnabled = true;
- executor = rsService.getExecutorService();
- }
+ this.oldestUnexpiredTS = now - scanInfo.getTtl();
+ this.minVersions = scanInfo.getMinVersions();
+
+ // We look up row-column Bloom filters for multi-column queries as part of
+ // the seek operation. However, we also look the row-column Bloom filter
+ // for multi-row (non-"get") scans because this is not done in
+ // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
+ this.useRowColBloom = numCol > 1 || (!get && numCol == 1);
+
+ this.maxRowSize = scanInfo.getTableMaxRowSize();
+ this.scanUsePread = scan.isSmall()? true: scanInfo.isUsePread();
+ this.cellsPerHeartbeatCheck = scanInfo.getCellsPerTimeoutCheck();
+ // Parallel seeking is on if the config allows and more there is more than one store file.
+ if (this.store != null && this.store.getStorefilesCount() > 1) {
+ RegionServerServices rsService = ((HStore)store).getHRegion().getRegionServerServices();
+ if (rsService != null && scanInfo.isParallelSeekEnabled()) {
+ this.parallelSeekEnabled = true;
+ this.executor = rsService.getExecutorService();
+ }
+ }
}
/**
@@ -198,12 +181,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
*/
public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet<byte[]> columns,
long readPt)
- throws IOException {
- this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
- scanInfo.getMinVersions(), readPt);
+ throws IOException {
+ this(store, scan, scanInfo, columns, readPt, scan.getCacheBlocks());
if (columns != null && scan.isRaw()) {
- throw new DoNotRetryIOException(
- "Cannot specify any column for a raw scan");
+ throw new DoNotRetryIOException("Cannot specify any column for a raw scan");
}
matcher = new ScanQueryMatcher(scan, scanInfo, columns,
ScanType.USER_SCAN, Long.MAX_VALUE, HConstants.LATEST_TIMESTAMP,
@@ -219,7 +200,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
// Always check bloom filter to optimize the top row seek for delete
// family marker.
seekScanners(scanners, matcher.getStartKey(), explicitColumnQuery
- && lazySeekEnabledGlobally, isParallelSeekEnabled);
+ && lazySeekEnabledGlobally, parallelSeekEnabled);
// set storeLimit
this.storeLimit = scan.getMaxResultsPerColumnFamily();
@@ -268,8 +249,8 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
private StoreScanner(Store store, ScanInfo scanInfo, Scan scan,
List<? extends KeyValueScanner> scanners, ScanType scanType, long smallestReadPoint,
long earliestPutTs, byte[] dropDeletesFromRow, byte[] dropDeletesToRow) throws IOException {
- this(store, false, scan, null, scanInfo.getTtl(), scanInfo.getMinVersions(),
- ((HStore)store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED));
+ this(store, scan, scanInfo, null,
+ ((HStore)store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED), false);
if (dropDeletesFromRow == null) {
matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType, smallestReadPoint,
earliestPutTs, oldestUnexpiredTS, now, store.getCoprocessorHost());
@@ -282,13 +263,13 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
scanners = selectScannersFrom(scanners);
// Seek all scanners to the initial key
- seekScanners(scanners, matcher.getStartKey(), false, isParallelSeekEnabled);
+ seekScanners(scanners, matcher.getStartKey(), false, parallelSeekEnabled);
// Combine all seeked scanners with a heap
resetKVHeap(scanners, store.getComparator());
}
- /** Constructor for testing. */
+ @VisibleForTesting
StoreScanner(final Scan scan, ScanInfo scanInfo,
ScanType scanType, final NavigableSet<byte[]> columns,
final List<KeyValueScanner> scanners) throws IOException {
@@ -298,7 +279,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
0);
}
- // Constructor for testing.
+ @VisibleForTesting
StoreScanner(final Scan scan, ScanInfo scanInfo,
ScanType scanType, final NavigableSet<byte[]> columns,
final List<KeyValueScanner> scanners, long earliestPutTs)
@@ -311,9 +292,8 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
public StoreScanner(final Scan scan, ScanInfo scanInfo,
ScanType scanType, final NavigableSet<byte[]> columns,
final List<KeyValueScanner> scanners, long earliestPutTs, long readPt)
- throws IOException {
- this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
- scanInfo.getMinVersions(), readPt);
+ throws IOException {
+ this(null, scan, scanInfo, columns, readPt, scan.getCacheBlocks());
this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType,
Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS, now, null);
@@ -322,7 +302,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
this.store.addChangedReaderObserver(this);
}
// Seek all scanners to the initial key
- seekScanners(scanners, matcher.getStartKey(), false, isParallelSeekEnabled);
+ seekScanners(scanners, matcher.getStartKey(), false, parallelSeekEnabled);
resetKVHeap(scanners, scanInfo.getComparator());
}
@@ -332,8 +312,8 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
*/
protected List<KeyValueScanner> getScannersNoCompaction() throws IOException {
final boolean isCompaction = false;
- boolean usePread = isGet || scanUsePread;
- return selectScannersFrom(store.getScanners(cacheBlocks, isGet, usePread,
+ boolean usePread = get || scanUsePread;
+ return selectScannersFrom(store.getScanners(cacheBlocks, get, usePread,
isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt));
}
@@ -452,7 +432,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
return;
}
if (withHeapClose) this.closing = true;
- // under test, we dont have a this.store
+ // Under test, we dont have a this.store
if (this.store != null) this.store.deleteChangedReaderObserver(this);
if (withHeapClose) {
for (KeyValueHeap h : this.heapsForDelayedClose) {
@@ -501,67 +481,65 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
@Override
public boolean next(List<Cell> outResult, ScannerContext scannerContext) throws IOException {
lock.lock();
-
try {
- if (scannerContext == null) {
- throw new IllegalArgumentException("Scanner context cannot be null");
- }
- if (checkReseek()) {
- return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
- }
+ if (scannerContext == null) {
+ throw new IllegalArgumentException("Scanner context cannot be null");
+ }
+ if (checkReseek()) {
+ return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
+ }
- // if the heap was left null, then the scanners had previously run out anyways, close and
- // return.
- if (this.heap == null) {
- // By this time partial close should happened because already heap is null
- close(false);// Do all cleanup except heap.close()
- return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
- }
+ // if the heap was left null, then the scanners had previously run out anyways, close and
+ // return.
+ if (this.heap == null) {
+ // By this time partial close should happened because already heap is null
+ close(false);// Do all cleanup except heap.close()
+ return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
+ }
- Cell cell = this.heap.peek();
- if (cell == null) {
- close(false);// Do all cleanup except heap.close()
- return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
- }
+ Cell cell = this.heap.peek();
+ if (cell == null) {
+ close(false);// Do all cleanup except heap.close()
+ return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
+ }
- // only call setRow if the row changes; avoids confusing the query matcher
- // if scanning intra-row
+ // only call setRow if the row changes; avoids confusing the query matcher
+ // if scanning intra-row
- // If no limits exists in the scope LimitScope.Between_Cells then we are sure we are changing
- // rows. Else it is possible we are still traversing the same row so we must perform the row
- // comparison.
- if (!scannerContext.hasAnyLimit(LimitScope.BETWEEN_CELLS) || matcher.curCell == null ||
- !CellUtil.matchingRow(cell, matcher.curCell)) {
- this.countPerRow = 0;
- matcher.setToNewRow(cell);
- }
+ // If no limits exists in the scope LimitScope.Between_Cells then we are sure we are changing
+ // rows. Else it is possible we are still traversing the same row so we must perform the row
+ // comparison.
+ if (!scannerContext.hasAnyLimit(LimitScope.BETWEEN_CELLS) || matcher.curCell == null ||
+ !CellUtil.matchingRow(cell, matcher.curCell)) {
+ this.countPerRow = 0;
+ matcher.setToNewRow(cell);
+ }
- // Clear progress away unless invoker has indicated it should be kept.
- if (!scannerContext.getKeepProgress()) scannerContext.clearProgress();
+ // Clear progress away unless invoker has indicated it should be kept.
+ if (!scannerContext.getKeepProgress()) scannerContext.clearProgress();
- // Only do a sanity-check if store and comparator are available.
- CellComparator comparator =
- store != null ? store.getComparator() : null;
+ // Only do a sanity-check if store and comparator are available.
+ CellComparator comparator = store != null ? store.getComparator() : null;
- int count = 0;
- long totalBytesRead = 0;
+ int count = 0;
+ long totalBytesRead = 0;
- LOOP: do {
- // Update and check the time limit based on the configured value of cellsPerTimeoutCheck
- if ((kvsScanned % cellsPerHeartbeatCheck == 0)) {
- scannerContext.updateTimeProgress();
- if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) {
- return scannerContext.setScannerState(NextState.TIME_LIMIT_REACHED).hasMoreValues();
+ LOOP: do {
+ // Update and check the time limit based on the configured value of cellsPerTimeoutCheck
+ if ((kvsScanned % cellsPerHeartbeatCheck == 0)) {
+ scannerContext.updateTimeProgress();
+ if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) {
+ return scannerContext.setScannerState(NextState.TIME_LIMIT_REACHED).hasMoreValues();
+ }
}
- }
- if (prevCell != cell) ++kvsScanned; // Do object compare - we set prevKV from the same heap.
- checkScanOrder(prevCell, cell, comparator);
- prevCell = cell;
+ if (prevCell != cell) ++kvsScanned; // Do object compare - we set prevKV from the same heap.
+ checkScanOrder(prevCell, cell, comparator);
+ prevCell = cell;
- ScanQueryMatcher.MatchCode qcode = matcher.match(cell);
- qcode = optimize(qcode, cell);
- switch(qcode) {
+ ScanQueryMatcher.MatchCode qcode = matcher.match(cell);
+ qcode = optimize(qcode, cell);
+ switch(qcode) {
case INCLUDE:
case INCLUDE_AND_SEEK_NEXT_ROW:
case INCLUDE_AND_SEEK_NEXT_COL:
@@ -597,7 +575,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
if (totalBytesRead > maxRowSize) {
throw new RowTooBigException("Max row size allowed: " + maxRowSize
- + ", but the row is bigger than that.");
+ + ", but the row is bigger than that.");
}
}
@@ -656,16 +634,16 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
default:
throw new RuntimeException("UNEXPECTED");
- }
- } while((cell = this.heap.peek()) != null);
+ }
+ } while((cell = this.heap.peek()) != null);
- if (count > 0) {
- return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
- }
+ if (count > 0) {
+ return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
+ }
- // No more keys
- close(false);// Do all cleanup except heap.close()
- return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
+ // No more keys
+ close(false);// Do all cleanup except heap.close()
+ return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
} finally {
lock.unlock();
}
@@ -764,7 +742,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
List<KeyValueScanner> scanners = getScannersNoCompaction();
// Seek all scanners to the initial key
- seekScanners(scanners, lastTopKey, false, isParallelSeekEnabled);
+ seekScanners(scanners, lastTopKey, false, parallelSeekEnabled);
// Combine all seeked scanners with a heap
resetKVHeap(scanners, store.getComparator());
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
index d6599c2..aba63c6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
@@ -391,7 +391,7 @@ public class TestPartitionedMobCompactor {
scan.setMaxVersions(hcd.getMaxVersions());
long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
long ttl = HStore.determineTTLFromFamily(hcd);
- ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
+ ScanInfo scanInfo = new ScanInfo(conf, hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
scanners, 0L, HConstants.LATEST_TIMESTAMP);
List<Cell> results = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
index d5e0221..b64a031 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -45,7 +45,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver {
public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
ScanInfo oldSI = store.getScanInfo();
- ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(),
+ ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getFamily(), oldSI.getTtl(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
Scan scan = new Scan();
scan.setMaxVersions(oldSI.getMaxVersions());
@@ -62,7 +62,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver {
InternalScanner s) throws IOException {
// this demonstrates how to override the scanners default behavior
ScanInfo oldSI = store.getScanInfo();
- ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(),
+ ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getFamily(), oldSI.getTtl(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
Scan scan = new Scan();
scan.setMaxVersions(oldSI.getMaxVersions());
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 54bf60e..6bdda43 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -195,7 +195,7 @@ public class TestCompaction {
for (Store hstore: this.r.stores.values()) {
HStore store = (HStore)hstore;
ScanInfo old = store.getScanInfo();
- ScanInfo si = new ScanInfo(old.getFamily(),
+ ScanInfo si = new ScanInfo(old.getConfiguration(), old.getFamily(),
old.getMinVersions(), old.getMaxVersions(), ttl,
old.getKeepDeletedCells(), 0, old.getComparator());
store.setScanInfo(si);
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index 50b1722..d68c6b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -312,7 +312,7 @@ public class TestDefaultCompactSelection extends TestCase {
public void testCompactionEmptyHFile() throws IOException {
// Set TTL
ScanInfo oldScanInfo = store.getScanInfo();
- ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
+ ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getConfiguration(), oldScanInfo.getFamily(),
oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
oldScanInfo.getComparator());
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index 066bbb3..882fd53 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -102,8 +102,9 @@ public class TestDefaultMemStore extends TestCase {
List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
Scan scan = new Scan();
List<Cell> result = new ArrayList<Cell>();
+ Configuration conf = HBaseConfiguration.create();
ScanInfo scanInfo =
- new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0,
+ new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0,
this.memstore.comparator);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
@@ -522,9 +523,10 @@ public class TestDefaultMemStore extends TestCase {
}
}
//starting from each row, validate results should contain the starting row
+ Configuration conf = HBaseConfiguration.create();
for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
- ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
- 0, this.memstore.comparator);
+ ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE,
+ KeepDeletedCells.FALSE, 0, this.memstore.comparator);
ScanType scanType = ScanType.USER_SCAN;
InternalScanner scanner = new StoreScanner(new Scan(
Bytes.toBytes(startRowId)), scanInfo, scanType, null,
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index ddd00d2..ede0793 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -286,7 +286,7 @@ public class TestMajorCompaction {
for (Store hstore : r.getStores()) {
HStore store = ((HStore) hstore);
ScanInfo old = store.getScanInfo();
- ScanInfo si = new ScanInfo(old.getFamily(),
+ ScanInfo si = new ScanInfo(old.getConfiguration(), old.getFamily(),
old.getMinVersions(), old.getMaxVersions(), ttl,
old.getKeepDeletedCells(), 0, old.getComparator());
store.setScanInfo(si);
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
index a01cc66..7cac301 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
@@ -442,7 +442,8 @@ public class TestMobStoreCompaction {
scan.setMaxVersions(hcd.getMaxVersions());
long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
long ttl = HStore.determineTTLFromFamily(hcd);
- ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
+ ScanInfo scanInfo = new ScanInfo(copyOfConf, hcd, ttl, timeToPurgeDeletes,
+ CellComparator.COMPARATOR);
StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
scanners, 0L, HConstants.LATEST_TIMESTAMP);
List<Cell> results = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index 9f27153..a4bcdee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -27,7 +27,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.NavigableSet;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
@@ -99,7 +101,7 @@ public class TestQueryMatcher extends HBaseTestCase {
private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
long now = EnvironmentEdgeManager.currentTime();
// 2,4,5
- ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
+ ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2),
now - ttl, now);
@@ -164,7 +166,7 @@ public class TestQueryMatcher extends HBaseTestCase {
expected.add(ScanQueryMatcher.MatchCode.DONE);
long now = EnvironmentEdgeManager.currentTime();
- ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
+ ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null,
now - ttl, now);
@@ -220,8 +222,9 @@ public class TestQueryMatcher extends HBaseTestCase {
long now = EnvironmentEdgeManager.currentTime();
ScanQueryMatcher qm =
- new ScanQueryMatcher(scan, new ScanInfo(fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0,
- rowComparator), get.getFamilyMap().get(fam2), now - testTTL, now);
+ new ScanQueryMatcher(scan,
+ new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0,
+ rowComparator), get.getFamilyMap().get(fam2), now - testTTL, now);
KeyValue [] kvs = new KeyValue[] {
new KeyValue(row1, fam2, col1, now-100, data),
@@ -274,7 +277,7 @@ public class TestQueryMatcher extends HBaseTestCase {
};
long now = EnvironmentEdgeManager.currentTime();
- ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
+ ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null,
now - testTTL, now);
@@ -331,7 +334,8 @@ public class TestQueryMatcher extends HBaseTestCase {
byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
long now = EnvironmentEdgeManager.currentTime();
// Set time to purge deletes to negative value to avoid it ever happening.
- ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
+ ScanInfo scanInfo =
+ new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
index c520422..c6cc902 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
@@ -253,7 +253,7 @@ public class TestReversibleScanners {
TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE);
ScanType scanType = ScanType.USER_SCAN;
- ScanInfo scanInfo = new ScanInfo(FAMILYNAME, 0, Integer.MAX_VALUE,
+ ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(), FAMILYNAME, 0, Integer.MAX_VALUE,
Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, CellComparator.COMPARATOR);
// Case 1.Test a full reversed scan
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
index bdd9feb..5b7e9cc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
@@ -30,8 +30,10 @@ import java.util.TreeSet;
import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
@@ -49,7 +51,8 @@ import org.junit.experimental.categories.Category;
public class TestStoreScanner extends TestCase {
private static final String CF_STR = "cf";
final byte [] CF = Bytes.toBytes(CF_STR);
- private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE,
+ static Configuration CONF = HBaseConfiguration.create();
+ private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE,
Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, CellComparator.COMPARATOR);
private ScanType scanType = ScanType.USER_SCAN;
@@ -418,7 +421,7 @@ public class TestStoreScanner extends TestCase {
List<KeyValueScanner> scanners = scanFixture(kvs);
Scan scan = new Scan();
scan.setMaxVersions(1);
- ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
+ ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
CellComparator.COMPARATOR);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner scanner =
@@ -489,7 +492,7 @@ public class TestStoreScanner extends TestCase {
Scan scan = new Scan();
scan.setMaxVersions(1);
// scanner with ttl equal to 500
- ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
+ ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
CellComparator.COMPARATOR);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner scanner =
@@ -549,7 +552,7 @@ public class TestStoreScanner extends TestCase {
List<KeyValueScanner> scanners = scanFixture(kvs);
Scan scan = new Scan();
scan.setMaxVersions(2);
- ScanInfo scanInfo = new ScanInfo(Bytes.toBytes("cf"),
+ ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
0 /* minVersions */,
2 /* maxVersions */, 500 /* ttl */,
KeepDeletedCells.FALSE /* keepDeletedCells */,
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
index 6ef6336..eb8513b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
@@ -189,7 +189,7 @@ public class TestStripeCompactor {
// Create store mock that is satisfactory for compactor.
HColumnDescriptor col = new HColumnDescriptor(NAME_OF_THINGS);
- ScanInfo si = new ScanInfo(col, Long.MAX_VALUE, 0, CellComparator.COMPARATOR);
+ ScanInfo si = new ScanInfo(conf, col, Long.MAX_VALUE, 0, CellComparator.COMPARATOR);
Store store = mock(Store.class);
when(store.getFamily()).thenReturn(col);
when(store.getScanInfo()).thenReturn(si);
http://git-wip-us.apache.org/repos/asf/hbase/blob/996866d3/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
index 451ccee..751130f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -250,7 +250,8 @@ public class TestCoprocessorScanPolicy {
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
- ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
+ ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(),
+ family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
@@ -270,7 +271,8 @@ public class TestCoprocessorScanPolicy {
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
- ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
+ ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(),
+ family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
@@ -290,7 +292,8 @@ public class TestCoprocessorScanPolicy {
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
- ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
+ ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(),
+ family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
@@ -301,5 +304,4 @@ public class TestCoprocessorScanPolicy {
}
}
}
-
-}
+}
\ No newline at end of file
[14/14] hbase git commit: HBASE-14684 Try to remove all
MiniMapReduceCluster in unit tests; REAPPLY
Posted by sy...@apache.org.
HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests; REAPPLY
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/89985760
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/89985760
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/89985760
Branch: refs/heads/hbase-12439
Commit: 899857609c7c2c2b0fd3fa72d3bd585cb2703e0a
Parents: 8a2cef3
Author: stack <st...@apache.org>
Authored: Mon Oct 26 08:40:19 2015 -0700
Committer: stack <st...@apache.org>
Committed: Mon Oct 26 08:40:32 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapred/TestTableInputFormat.java | 8 ++++----
.../apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java | 2 --
.../hbase/mapreduce/MultiTableInputFormatTestBase.java | 3 ---
.../hbase/mapreduce/TableSnapshotInputFormatTestBase.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestCellCounter.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestCopyTable.java | 2 --
.../apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 6 ------
.../hadoop/hbase/mapreduce/TestHFileOutputFormat2.java | 6 ------
.../hadoop/hbase/mapreduce/TestHRegionPartitioner.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestHashTable.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestImportExport.java | 2 --
.../mapreduce/TestImportTSVWithOperationAttributes.java | 2 --
.../apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java | 2 --
.../hbase/mapreduce/TestImportTSVWithVisibilityLabels.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestImportTsv.java | 2 --
.../hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestRowCounter.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestSyncTable.java | 2 --
.../apache/hadoop/hbase/mapreduce/TestTableInputFormat.java | 2 --
.../hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java | 3 ---
.../hadoop/hbase/mapreduce/TestTableMapReduceBase.java | 2 --
.../apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java | 2 --
.../org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java | 2 --
23 files changed, 4 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
index 8498341..1975c59 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
@@ -34,6 +34,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Connection;
@@ -80,7 +81,7 @@ public class TestTableInputFormat {
private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static MiniMRCluster mrCluster;
+
static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] columns = new byte[][] { FAMILY };
@@ -88,12 +89,10 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
- mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
@@ -344,7 +343,8 @@ public class TestTableInputFormat {
}
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
- final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
+ Configuration conf = UTIL.getConfiguration();
+ final JobConf job = new JobConf(conf);
job.setInputFormat(clazz);
job.setOutputFormat(NullOutputFormat.class);
job.setMapperClass(ExampleVerifier.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
index 628bb96..daa6e71 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
@@ -90,12 +90,10 @@ public class TestTableMapReduceUtil {
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
- UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
index 38e97b3..3cd8b4d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
@@ -83,13 +83,10 @@ public abstract class MultiTableInputFormatTestBase {
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
}
}
- // start MR cluster
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index d6c5d32..58db63e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -154,14 +154,12 @@ public abstract class TableSnapshotInputFormatTestBase {
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
- util.startMiniMapReduceCluster();
try {
Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
TableName tableName = TableName.valueOf("testWithMapReduce");
testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
expectedNumSplits, shutdownCluster);
} finally {
- util.shutdownMiniMapReduceCluster();
tearDownCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
index 78707fd..54a81b7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
@@ -63,14 +63,12 @@ public class TestCellCounter {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
- UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
FileUtil.fullyDelete(new File(OUTPUT_DIR));
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
index 1c13157..626383b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -62,12 +62,10 @@ public class TestCopyTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 90b92bb..25642fe 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -403,7 +403,6 @@ public class TestHFileOutputFormat {
assertEquals("Should make 5 regions", numRegions, 5);
// Generate the bulk load files
- util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table",
@@ -473,7 +472,6 @@ public class TestHFileOutputFormat {
assertEquals("Data should remain after reopening of regions",
tableDigestBefore, util.checksumRows(table));
} finally {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -937,7 +935,6 @@ public class TestHFileOutputFormat {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
- util.startMiniMapReduceCluster();
try (RegionLocator regionLocator = util.getConnection().getRegionLocator(TABLE_NAME)) {
for (int i = 0; i < 2; i++) {
@@ -978,7 +975,6 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -1019,7 +1015,6 @@ public class TestHFileOutputFormat {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
- util.startMiniMapReduceCluster();
try (RegionLocator locator = util.getConnection().getRegionLocator(TABLE_NAME)) {
runIncrementalPELoad(conf, table, locator, testDir);
@@ -1058,7 +1053,6 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index 7a13e5d..69b3276 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -430,7 +430,6 @@ public class TestHFileOutputFormat2 {
assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
// Generate the bulk load files
- util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table.getTableDescriptor(), r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@@ -511,7 +510,6 @@ public class TestHFileOutputFormat2 {
} finally {
testDir.getFileSystem(conf).delete(testDir, true);
util.deleteTable(TABLE_NAME);
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -976,7 +974,6 @@ public class TestHFileOutputFormat2 {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
- util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@@ -1018,7 +1015,6 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@@ -1060,7 +1056,6 @@ public class TestHFileOutputFormat2 {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
- util.startMiniMapReduceCluster();
RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@@ -1100,7 +1095,6 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
index 33d0e74..9b461d7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
@@ -36,12 +36,10 @@ public class TestHRegionPartitioner {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
- UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
index 2d03ac1..34020b5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
@@ -57,12 +57,10 @@ public class TestHashTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index a3fc4f6..b9ad6af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -111,14 +111,12 @@ public class TestImportExport {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.startMiniCluster();
- UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR =
new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
index 7360a63..66cf0ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
@@ -99,12 +99,10 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
util.startMiniCluster();
- util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
index 445620f..3fc129f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
@@ -90,12 +90,10 @@ public class TestImportTSVWithTTLs implements Configurable {
conf.setInt("hfile.format.version", 3);
conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
util.startMiniCluster();
- util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
index 3aec669..2b96d8c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
@@ -124,7 +124,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
// Wait for the labels table to become available
util.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
createLabels();
- util.startMiniMapReduceCluster();
}
private static void createLabels() throws IOException, InterruptedException {
@@ -148,7 +147,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
@AfterClass
public static void releaseCluster() throws Exception {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 66fdf93..9b7c4ae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -104,12 +104,10 @@ public class TestImportTsv implements Configurable {
@BeforeClass
public static void provisionCluster() throws Exception {
util.startMiniCluster();
- util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
- util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
index 9e1d2bf..03052fb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
@@ -76,13 +76,11 @@ public class TestMultithreadedTableMapper {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
- UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
index b0bf558..fe78844 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
@@ -64,7 +64,6 @@ public class TestRowCounter {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
- TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
@@ -76,7 +75,6 @@ public class TestRowCounter {
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
- TEST_UTIL.shutdownMiniMapReduceCluster();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
index 841d8cb..aca1705 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
@@ -60,12 +60,10 @@ public class TestSyncTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
index 0ec46e4..b0a4243 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
@@ -84,12 +84,10 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
- mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
index 23714b7..46e1a0e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
@@ -84,13 +84,10 @@ public abstract class TestTableInputFormatScanBase {
// create and fill table
table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
- // start MR cluster
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
index 5436a1d..77ebb48 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
@@ -79,12 +79,10 @@ public abstract class TestTableMapReduceBase {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
- UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
index 03da1ed..7e75830 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
@@ -168,7 +168,6 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
- UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");
@@ -185,7 +184,6 @@ public class TestTimeRangeMapRed {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
- UTIL.shutdownMiniMapReduceCluster();
if (job != null) {
FileUtil.fullyDelete(
new File(job.getConfiguration().get("hadoop.tmp.dir")));
http://git-wip-us.apache.org/repos/asf/hbase/blob/89985760/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index a12887e..60b050f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -73,12 +73,10 @@ public class TestWALPlayer {
@BeforeClass
public static void beforeClass() throws Exception {
cluster = TEST_UTIL.startMiniCluster();
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
[13/14] hbase git commit: HBASE-13867: Add endpoint coprocessor guide
to HBase book.
Posted by sy...@apache.org.
HBASE-13867: Add endpoint coprocessor guide to HBase book.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a2cef33
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a2cef33
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a2cef33
Branch: refs/heads/hbase-12439
Commit: 8a2cef3315516501627c7a30bdcf989b12a32303
Parents: 46c646d
Author: g-bhardwaj <ga...@gmail.com>
Authored: Sun Oct 25 17:29:35 2015 +0530
Committer: Misty Stanley-Jones <ms...@cloudera.com>
Committed: Mon Oct 26 09:06:06 2015 +1000
----------------------------------------------------------------------
src/main/asciidoc/_chapters/cp.adoc | 901 ++++++++++++++++++++++++++-----
1 file changed, 755 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a2cef33/src/main/asciidoc/_chapters/cp.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/cp.adoc b/src/main/asciidoc/_chapters/cp.adoc
index a99e903..06930d1 100644
--- a/src/main/asciidoc/_chapters/cp.adoc
+++ b/src/main/asciidoc/_chapters/cp.adoc
@@ -27,203 +27,812 @@
:icons: font
:experimental:
-HBase coprocessors are modeled after the coprocessors which are part of Google's BigTable (http://www.scribd.com/doc/21631448/Dean-Keynote-Ladis2009, pages 66-67.). Coprocessors function in a similar way to Linux kernel modules.
-They provide a way to run server-level code against locally-stored data.
-The functionality they provide is very powerful, but also carries great risk and can have adverse effects on the system, at the level of the operating system.
-The information in this chapter is primarily sourced and heavily reused from Mingjie Lai's blog post at https://blogs.apache.org/hbase/entry/coprocessor_introduction.
+HBase Coprocessors are modeled after the Coprocessors which are part of Google's BigTable
+(http://research.google.com/people/jeff/SOCC2010-keynote-slides.pdf pages 41-42.). +
+Coprocessor is a framework that provides an easy way to run your custom code directly on
+Region Server.
+The information in this chapter is primarily sourced and heavily reused from:
+
+. Mingjie Lai's blog post
+link:https://blogs.apache.org/hbase/entry/coprocessor_introduction[Coprocessor Introduction].
+. Gaurav Bhardwaj's blog post
+link:http://www.3pillarglobal.com/insights/hbase-coprocessors[The How To Of HBase Coprocessors].
+
-Coprocessors are not designed to be used by end users of HBase, but by HBase developers who need to add specialized functionality to HBase.
-One example of the use of coprocessors is pluggable compaction and scan policies, which are provided as coprocessors in link:https://issues.apache.org/jira/browse/HBASE-6427[HBASE-6427].
== Coprocessor Framework
-The implementation of HBase coprocessors diverges from the BigTable implementation.
-The HBase framework provides a library and runtime environment for executing user code within the HBase region server and master processes.
+When working with any data store (like RDBMS or HBase) you fetch the data (in case of RDBMS you
+might use SQL query and in case of HBase you use either Get or Scan). To fetch only relevant data
+you filter it (for RDBMS you put conditions in 'WHERE' predicate and in HBase you use
+link:http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/Filter.html[Filter]).
+After fetching the desired data, you perform your business computation on the data.
+This scenario is close to ideal for "small data", where few thousand rows and a bunch of columns
+are returned from the data store. Now imagine a scenario where there are billions of rows and
+millions of columns and you want to perform some computation which requires all the data, like
+calculating average or sum. Even if you are interested in just few columns, you still have to
+fetch all the rows. There are a few drawbacks in this approach as described below:
+
+. In this approach the data transfer (from data store to client side) will become the bottleneck,
+and the time required to complete the operation is limited by the rate at which data transfer
+takes place.
+. It's not always possible to hold so much data in memory and perform computation.
+. Bandwidth is one of the most precious resources in any data center. Operations like this may
+saturate your data center’s bandwidth and will severely impact the performance of your cluster.
+. Your client code is becoming thick as you are maintaining the code for calculating average or
+summation on client side. Not a major drawback when talking of severe issues like
+performance/bandwidth but still worth giving consideration.
+
+In a scenario like this it's better to move the computation (i.e. user's custom code) to the data
+itself (Region Server). Coprocessor helps you achieve this but you can do more than that.
+There is another advantage that your code runs in parallel (i.e. on all Regions).
+To give an idea of Coprocessor's capabilities, different people give different analogies.
+The three most famous analogies for Coprocessor are:
+[[cp_analogies]]
+Triggers and Stored Procedure:: This is the most common analogy for Coprocessor. Observer
+Coprocessor is compared to triggers because like triggers they execute your custom code when
+certain event occurs (like Get or Put etc.). Similarly Endpoints Coprocessor is compared to the
+stored procedures and you can perform custom computation on data directly inside the region server.
+
+MapReduce:: As in MapReduce you move the computation to the data in the same way. Coprocessor
+executes your custom computation directly on Region Servers, i.e. where data resides. That's why
+some people compare Coprocessor to a small MapReduce jobs.
+
+AOP:: Some people compare it to _Aspect Oriented Programming_ (AOP). As in AOP, you apply advice
+(on occurrence of specific event) by intercepting the request and then running some custom code
+(probably cross-cutting concerns) and then forwarding the request on its path as if nothing
+happened (or even return it back). Similarly in Coprocessor you have this facility of intercepting
+the request and running custom code and then forwarding it on its path (or returning it).
+
+
+Although Coprocessor derives its roots from Google's Bigtable but it deviates from it largely in
+its design. Currently there are efforts going on to bridge this gap. For more information see
+link:https://issues.apache.org/jira/browse/HBASE-4047[HBASE-4047].
+
+In HBase, to implement a Coprocessor certain steps must be followed as described below:
+
+. Either your class should extend one of the Coprocessor classes (like
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.html[BaseRegionObserver]
+) or it should implement Coprocessor interfaces (like
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/Coprocessor.html[Coprocessor],
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/CoprocessorService.html[CoprocessorService]).
+
+. Load the Coprocessor: Currently there are two ways to load the Coprocessor. +
+Static:: Loading from configuration
+Dynammic:: Loading via 'hbase shell' or via Java code using HTableDescriptor class). +
+For more details see <<cp_loading,Loading Coprocessors>>.
+
+. Finally your client-side code to call the Coprocessor. This is the easiest step, as HBase
+handles the Coprocessor transparently and you don't have to do much to call the Coprocessor.
+
+
+The framework API is provided in the
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/coprocessor/package-summary.html[coprocessor]
+package. +
+Coprocessors are not designed to be used by the end users but by developers. Coprocessors are
+executed directly on region server; therefore a faulty/malicious code can bring your region server
+down. Currently there is no mechanism to prevent this, but there are efforts going on for this.
+For more, see link:https://issues.apache.org/jira/browse/HBASE-4047[HBASE-4047]. +
+Two different types of Coprocessors are provided by the framework, based on their functionality.
+
+
+
+== Types of Coprocessors
+
+Coprocessor can be broadly divided into two categories: Observer and Endpoint.
+
+=== Observer
+Observer Coprocessor are easy to understand. People coming from RDBMS background can compare them
+to the triggers available in relational databases. Folks coming from programming background can
+visualize it like advice (before and after only) available in AOP (Aspect Oriented Programming).
+See <<cp_analogies, Coprocessor Analogy>> +
+Coprocessors allows you to hook your custom code in two places during the life cycle of an event. +
+First is just _before_ the occurrence of the event (just like 'before' advice in AOP or triggers
+like 'before update'). All methods providing this kind feature will start with the prefix `pre`. +
+For example if you want your custom code to get executed just before the `Put` operation, you can
+use the override the
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html#prePut%28org.apache.hadoop.hbase.coprocessor.ObserverContext,%20org.apache.hadoop.hbase.client.Put,%20org.apache.hadoop.hbase.regionserver.wal.WALEdit,%20org.apache.hadoop.hbase.client.Durability%29[`prePut`]
+method of
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html[RegionCoprocessor].
+This method has following signature:
+[source,java]
+----
+public void prePut (final ObserverContext e, final Put put, final WALEdit edit,final Durability
+durability) throws IOException;
+----
-The framework API is provided in the link:https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/coprocessor/package-summary.html[coprocessor] package.
+Secondly, the Observer Coprocessor also provides hooks for your code to get executed just _after_
+the occurrence of the event (similar to after advice in AOP terminology or 'after update' triggers
+). The methods giving this functionality will start with the prefix `post`. For example, if you
+want your code to be executed after the 'Put' operation, you should consider overriding
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html#postPut%28org.apache.hadoop.hbase.coprocessor.ObserverContext,%20org.apache.hadoop.hbase.client.Put,%20org.apache.hadoop.hbase.regionserver.wal.WALEdit,%20org.apache.hadoop.hbase.client.Durability%29[`postPut`]
+method of
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html[RegionCoprocessor]:
+[source,java]
+----
+public void postPut(final ObserverContext e, final Put put, final WALEdit edit, final Durability
+durability) throws IOException;
+----
+
+In short, the following conventions are generally followed: +
+Override _preXXX()_ method if you want your code to be executed just before the occurrence of the
+event. +
+Override _postXXX()_ method if you want your code to be executed just after the occurrence of the
+event. +
+
+.Use Cases for Observer Coprocessors:
+Few use cases of the Observer Coprocessor are:
+
+. *Security*: Before performing any operation (like 'Get', 'Put') you can check for permission in
+the 'preXXX' methods.
+
+. *Referential Integrity*: Unlike traditional RDBMS, HBase doesn't have the concept of referential
+integrity (foreign key). Suppose for example you have a requirement that whenever you insert a
+record in 'users' table, a corresponding entry should also be created in 'user_daily_attendance'
+table. One way you could solve this is by using two 'Put' one for each table, this way you are
+throwing the responsibility (of the referential integrity) to the user. A better way is to use
+Coprocessor and overriding 'postPut' method in which you write the code to insert the record in
+'user_daily_attendance' table. This way client code is more lean and clean.
+
+. *Secondary Index*: Coprocessor can be used to maintain secondary indexes. For more information
+see link:http://wiki.apache.org/hadoop/Hbase/SecondaryIndexing[SecondaryIndexing].
+
+
+==== Types of Observer Coprocessor
+
+Observer Coprocessor comes in following flavors:
+
+. *RegionObserver*: This Coprocessor provides the facility to hook your code when the events on
+region are triggered. Most common example include 'preGet' and 'postGet' for 'Get' operation and
+'prePut' and 'postPut' for 'Put' operation. For exhaustive list of supported methods (events) see
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html[RegionObserver].
+
+. *Region Server Observer*: Provides hook for the events related to the RegionServer, such as
+stopping the RegionServer and performing operations before or after merges, commits, or rollbacks.
+For more details please refer
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.html[RegionServerObserver].
+
+. *Master Observer*: This observer provides hooks for DDL like operation, such as create, delete,
+modify table. For entire list of available methods see
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/MasterObserver.html[MasterObserver].
+
+. *WAL Observer*: Provides hooks for WAL (Write-Ahead-Log) related operation. It has only two
+method 'preWALWrite()' and 'postWALWrite()'. For more details see
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/WALObserver.html[WALObserver].
+
+For example see <<cp_example,Examples>>
+
+
+=== Endpoint Coprocessor
+
+Endpoint Coprocessor can be compared to stored procedure found in RDBMS.
+See <<cp_analogies, Coprocessor Analogy>>. They help in performing computation which is not
+possible either through Observer Coprocessor or otherwise. For example, calculating average or
+summation over the entire table that spans across multiple regions. They do so by providing a hook
+for your custom code and then running it across all regions. +
+With Endpoints Coprocessor you can create your own dynamic RPC protocol and thus can provide
+communication between client and region server, hence enabling you to run your custom code on
+region server (on each region of a table). +
+Unlike observer Coprocessor (where your custom code is
+executed transparently when events like 'Get' operation occurs), in Endpoint Coprocessor you have
+to explicitly invoke the Coprocessor by using the
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/client/Table.html#coprocessorService%28java.lang.Class,%20byte%5B%5D,%20byte%5B%5D,%20org.apache.hadoop.hbase.client.coprocessor.Batch.Call%29[CoprocessorService()]
+method available in
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/client/Table.html[Table]
+(or
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/client/HTableInterface.html[HTableInterface]
+or
+link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/client/HTable.html[HTable]).
+
+From version 0.96, implementing Endpoint Coprocessor is not straight forward. Now it is done with
+the help of Google's Protocol Buffer. For more details on Protocol Buffer, please see
+link:https://developers.google.com/protocol-buffers/docs/proto[Protocol Buffer Guide].
+Endpoints Coprocessor written in version 0.94 are not compatible with with version 0.96 or later
+(for more details, see
+link:https://issues.apache.org/jira/browse/HBASE-5448[HBASE-5448]),
+so if your are upgrading your HBase cluster from version 0.94 (or before) to 0.96 (or later) you
+have to rewrite your Endpoint coprocessor.
+
+For example see <<cp_example,Examples>>
+
+
+[[cp_loading]]
+== Loading Coprocessors
+
+_Loading of Coprocessor refers to the process of making your custom Coprocessor implementation
+available to the the HBase, so that when a requests comes in or an event takes place the desired
+functionality implemented in your custom code gets executed. +
+Coprocessor can be loaded broadly in two ways. One is static (loading through configuration files)
+and the other one is dynamic loading (using hbase shell or java code).
+
+=== Static Loading
+Static loading means that your Coprocessor will take effect only when you restart your HBase and
+there is a reason for it. In this you make changes 'hbase-site.xml' and therefore have to restart
+HBase for your changes to take place. +
+Following are the steps for loading Coprocessor statically.
+
+. Define the Coprocessor in hbase-site.xml: Define a <property> element which consist of two
+sub elements <name> and <value> respectively.
++
+.. <name> can have one of the following values:
++
+... 'hbase.coprocessor.region.classes' for RegionObservers and Endpoints.
+... 'hbase.coprocessor.wal.classes' for WALObservers.
+... 'hbase.coprocessor.master.classes' for MasterObservers.
+.. <value> must contain the fully qualified class name of your class implmenting the Coprocessor.
++
+For example to load a Coprocessor (implemented in class SumEndPoint.java) you have to create
+following entry in RegionServer's 'hbase-site.xml' file (generally located under 'conf' directiory):
++
+[source,xml]
+----
+<property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value>org.myname.hbase.coprocessor.endpoint.SumEndPoint</value>
+</property>
+----
+If multiple classes are specified for loading, the class names must be comma-separated.
+The framework attempts to load all the configured classes using the default class loader.
+Therefore, the jar file must reside on the server-side HBase classpath.
++
+Coprocessors which are loaded in this way will be active on all regions of all tables.
+These are also called system Coprocessor.
+The first listed Coprocessors will be assigned the priority `Coprocessor.Priority.SYSTEM`.
+Each subsequent coprocessor in the list will have its priority value incremented by one (which
+reduces its priority, because priorities have the natural sort order of Integers).
++
+When calling out to registered observers, the framework executes their callbacks methods in the
+sorted order of their priority. +
+Ties are broken arbitrarily.
-Two different types of coprocessors are provided by the framework, based on their scope.
+. Put your code on classpth of HBase: There are various ways to do so, like adding jars on
+classpath etc. One easy way to do this is to drop the jar (containing you code and all the
+dependencies) in 'lib' folder of the HBase installation.
-.Types of Coprocessors
+. Restart the HBase.
-System Coprocessors::
- System coprocessors are loaded globally on all tables and regions hosted by a region server.
-Table Coprocessors::
- You can specify which coprocessors should be loaded on all regions for a table on a per-table basis.
+==== Unloading Static Coprocessor
+Unloading static Coprocessor is easy. Following are the steps:
-The framework provides two different aspects of extensions as well: _observers_ and _endpoints_.
+. Delete the Coprocessor's entry from the 'hbase-site.xml' i.e. remove the <property> tag.
-Observers::
- Observers are analogous to triggers in conventional databases.
- They allow you to insert user code by overriding upcall methods provided by the coprocessor framework.
- Callback functions are executed from core HBase code when events occur.
- Callbacks are handled by the framework, and the coprocessor itself only needs to insert the extended or alternate functionality.
+. Restart the Hbase.
-Endpoints (HBase 0.96.x and later)::
- The implementation for endpoints changed significantly in HBase 0.96.x due to the introduction of protocol buffers (protobufs) (link:https://issues.apache.org/jira/browse/HBASE-5448[HBASE-5488]). If you created endpoints before 0.96.x, you will need to rewrite them.
- Endpoints are now defined and callable as protobuf services, rather than endpoint invocations passed through as Writable blobs
+. Optionally remove the Coprocessor jar file from the classpath (or from the lib directory if you
+copied it over there). Removing the coprocessor JARs from HBase’s classpath is a good practice.
-Endpoints (HBase 0.94.x and earlier)::
- Dynamic RPC endpoints resemble stored procedures.
- An endpoint can be invoked at any time from the client.
- When it is invoked, it is executed remotely at the target region or regions, and results of the executions are returned to the client.
+=== Dynamic Loading
+Dynamic loading refers to the process of loading Coprocessor without restarting HBase. This may
+sound better than the static loading (and in some scenarios it may) but there is a caveat, dynamic
+loaded Coprocessor applies to the table only for which it was loaded while same is not true for
+static loading as it applies to all the tables. Due to this difference sometimes dynamically
+loaded Coprocessor are also called *Table Coprocessor* (as they applies only to a single table)
+while statically loaded Coprocessor are called *System Coprocessor* (as they applies to all the
+tables). +
+To dynamically load the Coprocessor you have to take the table offline hence during this time you
+won't be able to process any request involving this table. +
+There are three ways to dynamically load Coprocessor as shown below:
-== Examples
+[NOTE]
+.Assumptions
+====
+The below mentioned instructions makes the following assumptions:
-An example of an observer is included in _hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java_.
-Several endpoint examples are included in the same directory.
+* A JAR called `coprocessor.jar` contains the Coprocessor implementation along with all of its
+dependencies if any.
+* The JAR is available in HDFS in some location like
+`hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar`.
+====
-== Building A Coprocessor
+. *Using Shell*: You can load the Coprocessor using the HBase shell as follows:
+.. Disable Table: Take table offline by disabling it. Suppose if the table name is 'users', then
+to disable it enter following command:
++
+[source]
+----
+hbase(main):001:0> disable 'users'
+----
-Before you can build a processor, it must be developed, compiled, and packaged in a JAR file.
-The next step is to configure the coprocessor framework to use your coprocessor.
-You can load the coprocessor from your HBase configuration, so that the coprocessor starts with HBase, or you can configure the coprocessor from the HBase shell, as a table attribute, so that it is loaded dynamically when the table is opened or reopened.
+.. Load the Coprocessor: The Coprocessor jar should be on HDFS and should be accessible to HBase,
+to load the Coprocessor use following command:
++
+[source]
+----
+hbase(main):002:0> alter 'users', METHOD => 'table_att', 'Coprocessor'=>'hdfs://<namenode>:<port>/
+user/<hadoop-user>/coprocessor.jar| org.myname.hbase.Coprocessor.RegionObserverExample|1073741823|
+arg1=1,arg2=2'
+----
++
+The Coprocessor framework will try to read the class information from the coprocessor table
+attribute value.
+The value contains four pieces of information which are separated by the pipe (`|`) character.
++
+* File path: The jar file containing the Coprocessor implementation must be in a location where
+all region servers can read it. +
+You could copy the file onto the local disk on each region server, but it is recommended to store
+it in HDFS.
+* Class name: The full class name of the Coprocessor.
+* Priority: An integer. The framework will determine the execution sequence of all configured
+observers registered at the same hook using priorities. This field can be left blank. In that
+case the framework will assign a default priority value.
+* Arguments (Optional): This field is passed to the Coprocessor implementation. This is optional.
+
+.. Enable the table: To enable table type following command:
++
+----
+hbase(main):003:0> enable 'users'
+----
+.. Verification: This is optional but generally good practice to see if your Coprocessor is
+loaded successfully. Enter following command:
++
+----
+hbase(main):04:0> describe 'users'
+----
++
+You must see some output like this:
++
+----
+DESCRIPTION ENABLED
+'users', {TABLE_ATTRIBUTES => {coprocessor$1 => true 'hdfs://<namenode>:<port>/user/<hadoop-user>/
+coprocessor.jar| org.myname.hbase.Coprocessor.RegionObserverExample|1073741823|'}, {NAME =>
+'personalDet'.....
+----
-=== Load from Configuration
-To configure a coprocessor to be loaded when HBase starts, modify the RegionServer's _hbase-site.xml_ and configure one of the following properties, based on the type of observer you are configuring:
+. *Using setValue()* method of HTableDescriptor: This is done entirely in Java as follows:
++
+[source,java]
+----
+String tableName = "users";
+String path = "hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar";
+Configuration conf = HBaseConfiguration.create();
+HBaseAdmin admin = new HBaseAdmin(conf);
+admin.disableTable(tableName);
+HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
+HColumnDescriptor columnFamily1 = new HColumnDescriptor("personalDet");
+columnFamily1.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily1);
+HColumnDescriptor columnFamily2 = new HColumnDescriptor("salaryDet");
+columnFamily2.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily2);
+hTableDescriptor.setValue("COPROCESSOR$1", path + "|"
++ RegionObserverExample.class.getCanonicalName() + "|"
++ Coprocessor.PRIORITY_USER);
+admin.modifyTable(tableName, hTableDescriptor);
+admin.enableTable(tableName);
+----
-* `hbase.coprocessor.region.classes`for RegionObservers and Endpoints
-* `hbase.coprocessor.wal.classes`for WALObservers
-* `hbase.coprocessor.master.classes`for MasterObservers
+. *Using addCoprocessor()* method of HTableDescriptor: This method is available from 0.96 version
+onwards.
++
+[source,java]
+----
+String tableName = "users";
+String path = "hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar";
+Configuration conf = HBaseConfiguration.create();
+HBaseAdmin admin = new HBaseAdmin(conf);
+admin.disableTable(tableName);
+HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
+HColumnDescriptor columnFamily1 = new HColumnDescriptor("personalDet");
+columnFamily1.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily1);
+HColumnDescriptor columnFamily2 = new HColumnDescriptor("salaryDet");
+columnFamily2.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily2);
+hTableDescriptor.addCoprocessor(RegionObserverExample.class.getCanonicalName(), path,
+Coprocessor.PRIORITY_USER, null);
+admin.modifyTable(tableName, hTableDescriptor);
+admin.enableTable(tableName);
+----
-.Example RegionObserver Configuration
====
-In this example, one RegionObserver is configured for all the HBase tables.
+WARNING: There is no guarantee that the framework will load a given Coprocessor successfully.
+For example, the shell command neither guarantees a jar file exists at a particular location nor
+verifies whether the given class is actually contained in the jar file.
+====
-[source,xml]
+
+==== Unloading Dynamic Coprocessor
+. Using shell: Run following command from HBase shell to remove Coprocessor from a table.
++
+[source]
----
-<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>org.apache.hadoop.hbase.coprocessor.AggregateImplementation</value>
-</property>
+hbase(main):003:0> alter 'users', METHOD => 'table_att_unset',
+hbase(main):004:0* NAME => 'coprocessor$1'
----
-====
-If multiple classes are specified for loading, the class names must be comma-separated.
-The framework attempts to load all the configured classes using the default class loader.
-Therefore, the jar file must reside on the server-side HBase classpath.
+. Using HtableDescriptor: Simply reload the table definition _without_ setting the value of
+Coprocessor either in setValue() or addCoprocessor() methods. This will remove the Coprocessor
+attached to this table, if any. For example:
++
+[source,java]
+----
+String tableName = "users";
+String path = "hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar";
+Configuration conf = HBaseConfiguration.create();
+HBaseAdmin admin = new HBaseAdmin(conf);
+admin.disableTable(tableName);
+HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
+HColumnDescriptor columnFamily1 = new HColumnDescriptor("personalDet");
+columnFamily1.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily1);
+HColumnDescriptor columnFamily2 = new HColumnDescriptor("salaryDet");
+columnFamily2.setMaxVersions(3);
+hTableDescriptor.addFamily(columnFamily2);
+admin.modifyTable(tableName, hTableDescriptor);
+admin.enableTable(tableName);
+----
++
+Optionally you can also use removeCoprocessor() method of HTableDescriptor class.
-Coprocessors which are loaded in this way will be active on all regions of all tables.
-These are the system coprocessor introduced earlier.
-The first listed coprocessors will be assigned the priority `Coprocessor.Priority.SYSTEM`.
-Each subsequent coprocessor in the list will have its priority value incremented by one (which reduces its priority, because priorities have the natural sort order of Integers).
-When calling out to registered observers, the framework executes their callbacks methods in the sorted order of their priority.
-Ties are broken arbitrarily.
-=== Load from the HBase Shell
+[[cp_example]]
+== Examples
+HBase ships Coprocessor examples for Observer Coprocessor see
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/xref/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.html[ZooKeeperScanPolicyObserver]
+and for Endpoint Coprocessor see
+// Below URL is more than 100 characters long.
+link:http://hbase.apache.org/xref/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.html[RowCountEndpoint]
-You can load a coprocessor on a specific table via a table attribute.
-The following example will load the `FooRegionObserver` observer when table `t1` is read or re-read.
+A more detailed example is given below.
-.Load a Coprocessor On a Table Using HBase Shell
-====
+For the sake of example let's take an hypothetical case. Suppose there is a HBase table called
+'users'. The table has two column families 'personalDet' and 'salaryDet' containing personal
+details and salary details respectively. Below is the graphical representation of the 'users'
+table.
+
+.Users Table
+[width="100%",cols="7",options="header,footer"]
+|====================
+| 3+|personalDet 3+|salaryDet
+|*rowkey* |*name* |*lastname* |*dob* |*gross* |*net* |*allowances*
+|admin |Admin |Admin | 3+|
+|cdickens |Charles |Dickens |02/07/1812 |10000 |8000 |2000
+|jverne |Jules |Verne |02/08/1828 |12000 |9000 |3000
+|====================
+
+
+
+=== Observer Example
+For the purpose of demonstration of Coprocessor we are assuming that 'admin' is a special person
+and his details shouldn't be visible or returned to any client querying the 'users' table. +
+To implement this functionality we will take the help of Observer Coprocessor.
+Following are the implementation steps:
+
+. Write a class that extends the
+// Below URL is more than 100 characters long.
+link:https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.html[BaseRegionObserver]
+class.
+
+. Override the 'preGetOp()' method (Note that 'preGet()' method is now deprecated). The reason for
+overriding this method is to check if the client has queried for the rowkey with value 'admin' or
+not. If the client has queried rowkey with 'admin' value then return the call without allowing the
+system to perform the get operation thus saving on performance, otherwise process the request as
+normal.
+
+. Put your code and dependencies in the jar file.
+
+. Place the jar in HDFS where HBase can locate it.
+
+. Load the Coprocessor.
+
+. Write a simple program to test it.
+
+Following are the implementation of the above steps:
+
+. For Step 1 and Step 2, below is the code.
++
+[source,java]
----
-hbase(main):005:0> alter 't1', METHOD => 'table_att',
- 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2'
-Updating all regions with the new schema...
-1/1 regions updated.
-Done.
-0 row(s) in 1.0730 seconds
-
-hbase(main):006:0> describe 't1'
-DESCRIPTION ENABLED
- {NAME => 't1', coprocessor$1 => 'hdfs:///foo.jar|com.foo.FooRegio false
- nObserver|1001|arg1=1,arg2=2', FAMILIES => [{NAME => 'c1', DATA_B
- LOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE
- => '0', VERSIONS => '3', COMPRESSION => 'NONE', MIN_VERSIONS =>
- '0', TTL => '2147483647', KEEP_DELETED_CELLS => 'false', BLOCKSIZ
- E => '65536', IN_MEMORY => 'false', ENCODE_ON_DISK => 'true', BLO
- CKCACHE => 'true'}, {NAME => 'f1', DATA_BLOCK_ENCODING => 'NONE',
- BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3'
- , COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '2147483647'
- , KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY
- => 'false', ENCODE_ON_DISK => 'true', BLOCKCACHE => 'true'}]}
-1 row(s) in 0.0190 seconds
+public class RegionObserverExample extends BaseRegionObserver {
+
+ private static final byte[] ADMIN = Bytes.toBytes("admin");
+ private static final byte[] COLUMN_FAMILY = Bytes.toBytes("details");
+ private static final byte[] COLUMN = Bytes.toBytes("Admin_det");
+ private static final byte[] VALUE = Bytes.toBytes("You can't see Admin details");
+
+ @Override
+ public void preGetOp(final ObserverContext e, final Get get, final List results)
+ throws IOException {
+
+ if (Bytes.equals(get.getRow(),ADMIN)) {
+ Cell c = CellUtil.createCell(get.getRow(),COLUMN _FAMILY, COLUMN,
+ System.currentTimeMillis(), (byte)4, VALUE);
+ results.add(c);
+ e.bypass();
+ }
+
+ List kvs = new ArrayList(results.size());
+ for (Cell c : results) {
+ kvs.add(KeyValueUtil.ensureKeyValue(c));
+ }
+ preGet(e, get, kvs);
+ results.clear();
+ results.addAll(kvs);
+ }
+}
+----
+Overriding the 'preGetOp()' will only work for 'Get' operation. For 'Scan' operation it won't help
+you. To deal with it you have to override another method called 'preScannerOpen()' method, and
+add a Filter explicitly for admin as shown below:
++
+[source,java]
+----
+@Override
+public RegionScanner preScannerOpen(final ObserverContext e, final Scan scan,
+final RegionScanner s) throws IOException {
+
+ Filter filter = new RowFilter(CompareOp.NOT_EQUAL, new BinaryComparator(ADMIN));
+ scan.setFilter(filter);
+ return s;
+}
+----
++
+This method works but there is a _side effect_. If the client has used any Filter in his scan,
+then that Filter won't have any effect because our filter has replaced it. +
+Another option you can try is to deliberately remove the admin from result. This approach is
+shown below:
++
+[source,java]
+----
+@Override
+public boolean postScannerNext(final ObserverContext e, final InternalScanner s,
+final List results, final int limit, final boolean hasMore) throws IOException {
+ Result result = null;
+ Iterator iterator = results.iterator();
+ while (iterator.hasNext()) {
+ result = iterator.next();
+ if (Bytes.equals(result.getRow(), ROWKEY)) {
+ iterator.remove();
+ break;
+ }
+ }
+ return hasMore;
+}
----
-====
-The coprocessor framework will try to read the class information from the coprocessor table attribute value.
-The value contains four pieces of information which are separated by the `|` character.
+. Step 3: It's pretty convenient to export the above program in a jar file. Let's assume that was
+exported in a file called 'coprocessor.jar'.
-* File path: The jar file containing the coprocessor implementation must be in a location where all region servers can read it.
- You could copy the file onto the local disk on each region server, but it is recommended to store it in HDFS.
-* Class name: The full class name of the coprocessor.
-* Priority: An integer.
- The framework will determine the execution sequence of all configured observers registered at the same hook using priorities.
- This field can be left blank.
- In that case the framework will assign a default priority value.
-* Arguments: This field is passed to the coprocessor implementation.
+. Step 4: Copy the jar to HDFS. You may use command like this:
++
+[source]
+----
+hadoop fs -copyFromLocal coprocessor.jar coprocessor.jar
+----
-.Unload a Coprocessor From a Table Using HBase Shell
-====
+. Step 5: Load the Coprocessor, see <<cp_loading,Loading of Coprocessor>>.
+
+. Step 6: Run the following program to test. The first part is testing 'Get' and second 'Scan'.
++
+[source,java]
+----
+Configuration conf = HBaseConfiguration.create();
+// Use below code for HBase verion 1.x.x or above.
+Connection connection = ConnectionFactory.createConnection(conf);
+TableName tableName = TableName.valueOf("users");
+Table table = connection.getTable(tableName);
+
+//Use below code HBase verion 0.98.xx or below.
+//HConnection connection = HConnectionManager.createConnection(conf);
+//HTableInterface table = connection.getTable("users");
+
+Get get = new Get(Bytes.toBytes("admin"));
+Result result = table.get(get);
+for (Cell c : result.rawCells()) {
+ System.out.println(Bytes.toString(CellUtil.cloneRow(c))
+ + "==> " + Bytes.toString(CellUtil.cloneFamily(c))
+ + "{" + Bytes.toString(CellUtil.cloneQualifier(c))
+ + ":" + Bytes.toLong(CellUtil.cloneValue(c)) + "}");
+}
+Scan scan = new Scan();
+ResultScanner scanner = table.getScanner(scan);
+for (Result res : scanner) {
+ for (Cell c : res.rawCells()) {
+ System.out.println(Bytes.toString(CellUtil.cloneRow(c))
+ + " ==> " + Bytes.toString(CellUtil.cloneFamily(c))
+ + " {" + Bytes.toString(CellUtil.cloneQualifier(c))
+ + ":" + Bytes.toLong(CellUtil.cloneValue(c))
+ + "}");
+ }
+}
----
-hbase(main):007:0> alter 't1', METHOD => 'table_att_unset',
-hbase(main):008:0* NAME => 'coprocessor$1'
-Updating all regions with the new schema...
-1/1 regions updated.
-Done.
-0 row(s) in 1.1130 seconds
-
-hbase(main):009:0> describe 't1'
-DESCRIPTION ENABLED
- {NAME => 't1', FAMILIES => [{NAME => 'c1', DATA_BLOCK_ENCODING => false
- 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSION
- S => '3', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '214
- 7483647', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN
- _MEMORY => 'false', ENCODE_ON_DISK => 'true', BLOCKCACHE => 'true
- '}, {NAME => 'f1', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER =>
- 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3', COMPRESSION =>
- 'NONE', MIN_VERSIONS => '0', TTL => '2147483647', KEEP_DELETED_C
- ELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY => 'false', ENCO
- DE_ON_DISK => 'true', BLOCKCACHE => 'true'}]}
-1 row(s) in 0.0180 seconds
+=== Endpoint Example
+
+In our hypothetical example (See Users Table), to demonstrate the Endpoint Coprocessor we see a
+trivial use case in which we will try to calculate the total (Sum) of gross salary of all
+employees. One way of implementing Endpoint Coprocessor (for version 0.96 and above) is as follows:
+
+. Create a '.proto' file defining your service.
+
+. Execute the 'protoc' command to generate the Java code from the above '.proto' file.
+
+. Write a class that should:
+.. Extend the above generated service class.
+.. It should also implement two interfaces Coprocessor and CoprocessorService.
+.. Override the service method.
+
+. Load the Coprocessor.
+
+. Write a client code to call Coprocessor.
+
+Implementation detail of the above steps is as follows:
+
+. Step 1: Create a 'proto' file to define your service, request and response. Let's call this file
+"sum.proto". Below is the content of the 'sum.proto' file.
++
+[source]
+----
+option java_package = "org.myname.hbase.coprocessor.autogenerated";
+option java_outer_classname = "Sum";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+message SumRequest {
+ required string family = 1;
+ required string column = 2;
+}
+
+message SumResponse {
+ required int64 sum = 1 [default = 0];
+}
+
+service SumService {
+ rpc getSum(SumRequest)
+ returns (SumResponse);
+}
----
-====
-WARNING: There is no guarantee that the framework will load a given coprocessor successfully.
-For example, the shell command neither guarantees a jar file exists at a particular location nor verifies whether the given class is actually contained in the jar file.
+. Step 2: Compile the proto file using proto compiler (for detailed instructions see the
+link:https://developers.google.com/protocol-buffers/docs/overview[official documentation]).
++
+[source]
+----
+$ protoc --java_out=src ./sum.proto
+----
++
+[note]
+----
+(Note: It is necessary for you to create the src folder).
+This will generate a class call "Sum.java".
+----
-== Check the Status of a Coprocessor
+. Step 3: Write your Endpoint Coprocessor: Firstly your class should extend the service just
+defined above (i.e. Sum.SumService). Second it should implement Coprocessor and CoprocessorService
+interfaces. Third, override the 'getService()', 'start()', 'stop()' and 'getSum()' methods.
+Below is the full code:
++
+[source,java]
+----
+public class SumEndPoint extends SumService implements Coprocessor, CoprocessorService {
+
+ private RegionCoprocessorEnvironment env;
+
+ @Override
+ public Service getService() {
+ return this;
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ if (env instanceof RegionCoprocessorEnvironment) {
+ this.env = (RegionCoprocessorEnvironment)env;
+ } else {
+ throw new CoprocessorException("Must be loaded on a table region!");
+ }
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment env) throws IOException {
+ // do mothing
+ }
+
+ @Override
+ public void getSum(RpcController controller, SumRequest request, RpcCallback done) {
+ Scan scan = new Scan();
+ scan.addFamily(Bytes.toBytes(request.getFamily()));
+ scan.addColumn(Bytes.toBytes(request.getFamily()), Bytes.toBytes(request.getColumn()));
+ SumResponse response = null;
+ InternalScanner scanner = null;
+ try {
+ scanner = env.getRegion().getScanner(scan);
+ List results = new ArrayList();
+ boolean hasMore = false;
+ long sum = 0L;
+ do {
+ hasMore = scanner.next(results);
+ for (Cell cell : results) {
+ sum = sum + Bytes.toLong(CellUtil.cloneValue(cell));
+ }
+ results.clear();
+ } while (hasMore);
+
+ response = SumResponse.newBuilder().setSum(sum).build();
+
+ } catch (IOException ioe) {
+ ResponseConverter.setControllerException(controller, ioe);
+ } finally {
+ if (scanner != null) {
+ try {
+ scanner.close();
+ } catch (IOException ignored) {}
+ }
+ }
+ done.run(response);
+ }
+}
+----
-To check the status of a coprocessor after it has been configured, use the `status` HBase Shell command.
+. Step 4: Load the Coprocessor. See <<cp_loading,loading of Coprocessor>>.
+. Step 5: Now we have to write the client code to test it. To do so in your main method, write the
+following code as shown below:
++
+[source,java]
----
-hbase(main):020:0> status 'detailed'
-version 0.92-tm-6
-0 regionsInTransition
-master coprocessors: []
-1 live servers
- localhost:52761 1328082515520
- requestsPerSecond=3, numberOfOnlineRegions=3, usedHeapMB=32, maxHeapMB=995
- -ROOT-,,0
- numberOfStores=1, numberOfStorefiles=1, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0,
-storefileIndexSizeMB=0, readRequestsCount=54, writeRequestsCount=1, rootIndexSizeKB=0, totalStaticIndexSizeKB=0,
-totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[]
- .META.,,1
- numberOfStores=1, numberOfStorefiles=0, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0,
-storefileIndexSizeMB=0, readRequestsCount=97, writeRequestsCount=4, rootIndexSizeKB=0, totalStaticIndexSizeKB=0,
-totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[]
- t1,,1328082575190.c0491168a27620ffe653ec6c04c9b4d1.
- numberOfStores=2, numberOfStorefiles=1, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0,
-storefileIndexSizeMB=0, readRequestsCount=0, writeRequestsCount=0, rootIndexSizeKB=0, totalStaticIndexSizeKB=0,
-totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN,
-coprocessors=[AggregateImplementation]
-0 dead servers
+Configuration conf = HBaseConfiguration.create();
+// Use below code for HBase verion 1.x.x or above.
+Connection connection = ConnectionFactory.createConnection(conf);
+TableName tableName = TableName.valueOf("users");
+Table table = connection.getTable(tableName);
+
+//Use below code HBase verion 0.98.xx or below.
+//HConnection connection = HConnectionManager.createConnection(conf);
+//HTableInterface table = connection.getTable("users");
+
+final SumRequest request = SumRequest.newBuilder().setFamily("salaryDet").setColumn("gross")
+ .build();
+try {
+Map<byte[], Long> results = table.CoprocessorService (SumService.class, null, null,
+new Batch.Call<SumService, Long>() {
+ @Override
+ public Long call(SumService aggregate) throws IOException {
+BlockingRpcCallback rpcCallback = new BlockingRpcCallback();
+ aggregate.getSum(null, request, rpcCallback);
+ SumResponse response = rpcCallback.get();
+ return response.hasSum() ? response.getSum() : 0L;
+ }
+ });
+ for (Long sum : results.values()) {
+ System.out.println("Sum = " + sum);
+ }
+} catch (ServiceException e) {
+e.printStackTrace();
+} catch (Throwable e) {
+ e.printStackTrace();
+}
----
+
== Monitor Time Spent in Coprocessors
-HBase 0.98.5 introduced the ability to monitor some statistics relating to the amount of time spent executing a given coprocessor.
-You can see these statistics via the HBase Metrics framework (see <<hbase_metrics>> or the Web UI for a given Region Server, via the _Coprocessor Metrics_ tab.
-These statistics are valuable for debugging and benchmarking the performance impact of a given coprocessor on your cluster.
+HBase 0.98.5 introduced the ability to monitor some statistics relating to the amount of time
+spent executing a given Coprocessor.
+You can see these statistics via the HBase Metrics framework (see <<hbase_metrics>> or the Web UI
+for a given Region Server, via the _Coprocessor Metrics_ tab.
+These statistics are valuable for debugging and benchmarking the performance impact of a given
+Coprocessor on your cluster.
Tracked statistics include min, max, average, and 90th, 95th, and 99th percentile.
All times are shown in milliseconds.
-The statistics are calculated over coprocessor execution samples recorded during the reporting interval, which is 10 seconds by default.
+The statistics are calculated over Coprocessor execution samples recorded during the reporting
+interval, which is 10 seconds by default.
The metrics sampling rate as described in <<hbase_metrics>>.
.Coprocessor Metrics UI
[12/14] hbase git commit: HBASE-14648 Reenable
TestWALProcedureStoreOnHDFS#testWalRollOnLowReplication (Heng Chen)
Posted by sy...@apache.org.
HBASE-14648 Reenable TestWALProcedureStoreOnHDFS#testWalRollOnLowReplication (Heng Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/46c646d6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/46c646d6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/46c646d6
Branch: refs/heads/hbase-12439
Commit: 46c646d6154d6a5291c308e065802248f4e4e7b9
Parents: 9597847
Author: stack <st...@apache.org>
Authored: Sat Oct 24 20:55:37 2015 -0700
Committer: stack <st...@apache.org>
Committed: Sat Oct 24 20:55:47 2015 -0700
----------------------------------------------------------------------
.../procedure/TestWALProcedureStoreOnHDFS.java | 167 +++++++++----------
1 file changed, 79 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/46c646d6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
index b360966..2e80b69 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
@@ -66,7 +66,7 @@ public class TestWALProcedureStoreOnHDFS {
}
};
- private static void setupConf(Configuration conf) {
+ private static void initConfig(Configuration conf) {
conf.setInt("dfs.replication", 3);
conf.setInt("dfs.namenode.replication.min", 3);
@@ -76,20 +76,16 @@ public class TestWALProcedureStoreOnHDFS {
conf.setInt("hbase.procedure.store.wal.sync.failure.roll.max", 10);
}
- @Before
public void setup() throws Exception {
- setupConf(UTIL.getConfiguration());
MiniDFSCluster dfs = UTIL.startMiniDFSCluster(3);
Path logDir = new Path(new Path(dfs.getFileSystem().getUri()), "/test-logs");
- store = ProcedureTestingUtility.createWalStore(
- UTIL.getConfiguration(), dfs.getFileSystem(), logDir);
+ store = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), dfs.getFileSystem(), logDir);
store.registerListener(stopProcedureListener);
store.start(8);
store.recoverLease();
}
- @After
public void tearDown() throws Exception {
store.stop(false);
UTIL.getDFSCluster().getFileSystem().delete(store.getLogDir(), true);
@@ -103,107 +99,102 @@ public class TestWALProcedureStoreOnHDFS {
@Test(timeout=60000, expected=RuntimeException.class)
public void testWalAbortOnLowReplication() throws Exception {
- assertEquals(3, UTIL.getDFSCluster().getDataNodes().size());
-
- LOG.info("Stop DataNode");
- UTIL.getDFSCluster().stopDataNode(0);
- assertEquals(2, UTIL.getDFSCluster().getDataNodes().size());
+ initConfig(UTIL.getConfiguration());
+ setup();
+ try {
+ assertEquals(3, UTIL.getDFSCluster().getDataNodes().size());
- store.insert(new TestProcedure(1, -1), null);
- for (long i = 2; store.isRunning(); ++i) {
+ LOG.info("Stop DataNode");
+ UTIL.getDFSCluster().stopDataNode(0);
assertEquals(2, UTIL.getDFSCluster().getDataNodes().size());
- store.insert(new TestProcedure(i, -1), null);
- Thread.sleep(100);
+
+ store.insert(new TestProcedure(1, -1), null);
+ for (long i = 2; store.isRunning(); ++i) {
+ assertEquals(2, UTIL.getDFSCluster().getDataNodes().size());
+ store.insert(new TestProcedure(i, -1), null);
+ Thread.sleep(100);
+ }
+ assertFalse(store.isRunning());
+ fail("The store.insert() should throw an exeption");
+ } finally {
+ tearDown();
}
- assertFalse(store.isRunning());
- fail("The store.insert() should throw an exeption");
}
@Test(timeout=60000)
public void testWalAbortOnLowReplicationWithQueuedWriters() throws Exception {
- assertEquals(3, UTIL.getDFSCluster().getDataNodes().size());
- store.registerListener(new ProcedureStore.ProcedureStoreListener() {
- @Override
- public void postSync() {
- Threads.sleepWithoutInterrupt(2000);
- }
+ initConfig(UTIL.getConfiguration());
+ setup();
+ try {
+ assertEquals(3, UTIL.getDFSCluster().getDataNodes().size());
+ store.registerListener(new ProcedureStore.ProcedureStoreListener() {
+ @Override
+ public void postSync() {
+ Threads.sleepWithoutInterrupt(2000);
+ }
- @Override
- public void abortProcess() {}
- });
-
- final AtomicInteger reCount = new AtomicInteger(0);
- Thread[] thread = new Thread[store.getNumThreads() * 2 + 1];
- for (int i = 0; i < thread.length; ++i) {
- final long procId = i + 1;
- thread[i] = new Thread() {
- public void run() {
- try {
- LOG.debug("[S] INSERT " + procId);
- store.insert(new TestProcedure(procId, -1), null);
- LOG.debug("[E] INSERT " + procId);
- } catch (RuntimeException e) {
- reCount.incrementAndGet();
- LOG.debug("[F] INSERT " + procId + ": " + e.getMessage());
+ @Override
+ public void abortProcess() {}
+ });
+
+ final AtomicInteger reCount = new AtomicInteger(0);
+ Thread[] thread = new Thread[store.getNumThreads() * 2 + 1];
+ for (int i = 0; i < thread.length; ++i) {
+ final long procId = i + 1;
+ thread[i] = new Thread() {
+ public void run() {
+ try {
+ LOG.debug("[S] INSERT " + procId);
+ store.insert(new TestProcedure(procId, -1), null);
+ LOG.debug("[E] INSERT " + procId);
+ } catch (RuntimeException e) {
+ reCount.incrementAndGet();
+ LOG.debug("[F] INSERT " + procId + ": " + e.getMessage());
+ }
}
- }
- };
- thread[i].start();
- }
+ };
+ thread[i].start();
+ }
- Thread.sleep(1000);
- LOG.info("Stop DataNode");
- UTIL.getDFSCluster().stopDataNode(0);
- assertEquals(2, UTIL.getDFSCluster().getDataNodes().size());
+ Thread.sleep(1000);
+ LOG.info("Stop DataNode");
+ UTIL.getDFSCluster().stopDataNode(0);
+ assertEquals(2, UTIL.getDFSCluster().getDataNodes().size());
- for (int i = 0; i < thread.length; ++i) {
- thread[i].join();
- }
+ for (int i = 0; i < thread.length; ++i) {
+ thread[i].join();
+ }
- assertFalse(store.isRunning());
- assertTrue(reCount.toString(), reCount.get() >= store.getNumThreads() &&
- reCount.get() < thread.length);
+ assertFalse(store.isRunning());
+ assertTrue(reCount.toString(), reCount.get() >= store.getNumThreads() &&
+ reCount.get() < thread.length);
+ } finally {
+ tearDown();
+ }
}
- @Ignore ("Needs work") @Test(timeout=60000)
+ @Test(timeout=60000)
public void testWalRollOnLowReplication() throws Exception {
- store.unregisterListener(stopProcedureListener);
- store.registerListener(new ProcedureStore.ProcedureStoreListener() {
- @Override
- public void postSync() {}
-
- @Override
- public void abortProcess() {
- LOG.info("Aborted!!!!");
- }
- });
- int dnCount = 0;
- store.insert(new TestProcedure(1, -1), null);
- UTIL.getDFSCluster().restartDataNode(dnCount);
- for (long i = 2; i < 100; ++i) {
- try {
+ initConfig(UTIL.getConfiguration());
+ UTIL.getConfiguration().setInt("dfs.namenode.replication.min", 1);
+ setup();
+ try {
+ int dnCount = 0;
+ store.insert(new TestProcedure(1, -1), null);
+ UTIL.getDFSCluster().restartDataNode(dnCount);
+ for (long i = 2; i < 100; ++i) {
store.insert(new TestProcedure(i, -1), null);
- } catch (RuntimeException re) {
- String msg = re.getMessage();
- // We could get a sync failed here...if the test cluster is crawling such that DN recovery
- // is taking a long time. If we've done enough passes, just finish up the test as a 'pass'
- if (msg != null && msg.toLowerCase().contains("sync aborted")) {
- LOG.info("i=" + i, re);
- if (i > 50) {
- LOG.info("Returning early... i=" + i + "...We ran enough of this test", re);
- return;
- }
+ waitForNumReplicas(3);
+ Thread.sleep(100);
+ if ((i % 30) == 0) {
+ LOG.info("Restart Data Node");
+ UTIL.getDFSCluster().restartDataNode(++dnCount % 3);
}
- throw re;
- }
- waitForNumReplicas(3);
- Thread.sleep(100);
- if ((i % 30) == 0) {
- LOG.info("Restart Data Node");
- UTIL.getDFSCluster().restartDataNode(++dnCount % 3);
}
+ assertTrue(store.isRunning());
+ } finally {
+ tearDown();
}
- assertTrue(store.isRunning());
}
public void waitForNumReplicas(int numReplicas) throws Exception {
[06/14] hbase git commit: HBASE-14690 Fix css so there's no
left/right scroll bar
Posted by sy...@apache.org.
HBASE-14690 Fix css so there's no left/right scroll bar
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9925e9a1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9925e9a1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9925e9a1
Branch: refs/heads/hbase-12439
Commit: 9925e9a182b4d6802d3587b65de9d3c6f98cd963
Parents: 604ff56
Author: Elliott Clark <ec...@apache.org>
Authored: Sat Oct 24 10:14:18 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Sat Oct 24 10:27:12 2015 -0700
----------------------------------------------------------------------
.../src/main/resources/hbase-webapps/static/css/hbase.css | 1 +
.../src/main/resources/hbase-webapps/static/css/hbase.css | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9925e9a1/hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css b/hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css
index f3bd9e9..03d8ff4 100644
--- a/hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css
+++ b/hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css
@@ -20,6 +20,7 @@
body { padding-top: 50px; }
html .content {
margin-left: 2.0em;
+ margin-right: 2.0em;
}
.navbar-brand img {
margin-top: -7px;
http://git-wip-us.apache.org/repos/asf/hbase/blob/9925e9a1/hbase-thrift/src/main/resources/hbase-webapps/static/css/hbase.css
----------------------------------------------------------------------
diff --git a/hbase-thrift/src/main/resources/hbase-webapps/static/css/hbase.css b/hbase-thrift/src/main/resources/hbase-webapps/static/css/hbase.css
index 4b36c80..03d8ff4 100644
--- a/hbase-thrift/src/main/resources/hbase-webapps/static/css/hbase.css
+++ b/hbase-thrift/src/main/resources/hbase-webapps/static/css/hbase.css
@@ -19,7 +19,8 @@
/* General styling */
body { padding-top: 50px; }
html .content {
- margin-left: 1.0em;
+ margin-left: 2.0em;
+ margin-right: 2.0em;
}
.navbar-brand img {
margin-top: -7px;