You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2017/09/28 07:49:55 UTC
[2/3] hbase git commit: HBASE-18826 Use HStore instead of Store in
our own code base and remove unnecessary methods in Store interface
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
index d558307..dec28f3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
@@ -53,11 +53,11 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
-import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -65,9 +65,9 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.Ignore;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
@@ -184,8 +184,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region = (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName)
+ .getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -274,8 +275,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -332,8 +334,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -393,7 +396,8 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
BlockCache cache = setCacheProperties(region);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, data);
@@ -485,7 +489,8 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
BlockCache cache = setCacheProperties(region);
Put put = new Put(ROW);
@@ -568,8 +573,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setEvictOnClose(true);
BlockCache cache = cacheConf.getBlockCache();
@@ -626,8 +632,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -708,7 +715,8 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
BlockCache cache = setCacheProperties(region);
Put put = new Put(ROW);
@@ -777,11 +785,11 @@ public class TestBlockEvictionFromClient {
}
}
- private BlockCache setCacheProperties(Region region) {
- Iterator<? extends Store> strItr = region.getStores().iterator();
+ private BlockCache setCacheProperties(HRegion region) {
+ Iterator<HStore> strItr = region.getStores().iterator();
BlockCache cache = null;
while (strItr.hasNext()) {
- Store store = strItr.next();
+ HStore store = strItr.next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -807,8 +815,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -872,8 +881,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -989,8 +999,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -1118,8 +1129,9 @@ public class TestBlockEvictionFromClient {
// get the block cache and region
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region =
+ (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -1446,14 +1458,14 @@ public class TestBlockEvictionFromClient {
}
}
- private void waitForStoreFileCount(Store store, int count, int timeout)
+ private void waitForStoreFileCount(HStore store, int count, int timeout)
throws InterruptedException {
long start = System.currentTimeMillis();
while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
Thread.sleep(100);
}
- System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur="
- + store.getStorefilesCount());
+ System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" +
+ store.getStorefilesCount());
assertEquals(count, store.getStorefilesCount());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index a938db6..400e109 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -199,19 +199,19 @@ public class TestFromClientSide {
try {
Append append = new Append(ROW);
- append.addColumn(TEST_UTIL.fam1, QUALIFIER, VALUE);
+ append.addColumn(HBaseTestingUtility.fam1, QUALIFIER, VALUE);
Result result = table.append(append);
// Verify expected result
Cell[] cells = result.rawCells();
assertEquals(1, cells.length);
- assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
+ assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
// Verify expected result again
Result readResult = table.get(new Get(ROW));
cells = readResult.rawCells();
assertEquals(1, cells.length);
- assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
+ assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
} finally {
table.close();
connection.close();
@@ -568,7 +568,7 @@ public class TestFromClientSide {
}
@Override
- protected List<KeyValueScanner> selectScannersFrom(Store store,
+ protected List<KeyValueScanner> selectScannersFrom(HStore store,
List<? extends KeyValueScanner> allScanners) {
List<KeyValueScanner> scanners = super.selectScannersFrom(store, allScanners);
List<KeyValueScanner> newScanners = new ArrayList<>(scanners.size());
@@ -596,7 +596,8 @@ public class TestFromClientSide {
public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s,
final long readPt) throws IOException {
- return new MyStoreScanner((HStore) store, store.getScanInfo(), scan, targetCols, readPt);
+ HStore hs = (HStore) store;
+ return new MyStoreScanner(hs, hs.getScanInfo(), scan, targetCols, readPt);
}
}
@@ -2043,7 +2044,6 @@ public class TestFromClientSide {
public void testDeleteWithFailed() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
- byte [][] ROWS = makeNAscii(ROW, 6);
byte [][] FAMILIES = makeNAscii(FAMILY, 3);
byte [][] VALUES = makeN(VALUE, 5);
long [] ts = {1000, 2000, 3000, 4000, 5000};
@@ -2061,7 +2061,7 @@ public class TestFromClientSide {
Get get = new Get(ROW);
get.addFamily(FAMILIES[0]);
- get.setMaxVersions(Integer.MAX_VALUE);
+ get.readAllVersions();
Result result = ht.get(get);
assertTrue(Bytes.equals(result.getValue(FAMILIES[0], QUALIFIER), VALUES[0]));
}
@@ -5301,8 +5301,9 @@ public class TestFromClientSide {
// get the block cache and region
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
- Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
- Store store = region.getStores().iterator().next();
+ HRegion region = (HRegion) TEST_UTIL.getRSForFirstRegionInTable(tableName)
+ .getRegion(regionName);
+ HStore store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
@@ -5394,15 +5395,14 @@ public class TestFromClientSide {
}
}
- private void waitForStoreFileCount(Store store, int count, int timeout)
- throws InterruptedException {
+ private void waitForStoreFileCount(HStore store, int count, int timeout)
+ throws InterruptedException {
long start = System.currentTimeMillis();
- while (start + timeout > System.currentTimeMillis() &&
- store.getStorefilesCount() != count) {
+ while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
Thread.sleep(100);
}
- System.out.println("start=" + start + ", now=" +
- System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
+ System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" +
+ store.getStorefilesCount());
assertEquals(count, store.getStorefilesCount());
}
@@ -5462,8 +5462,8 @@ public class TestFromClientSide {
// Test Initialization.
byte [] startKey = Bytes.toBytes("ddc");
byte [] endKey = Bytes.toBytes("mmm");
- final TableName tableName = TableName.valueOf(name.getMethodName());
- Table t = TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
+ TableName tableName = TableName.valueOf(name.getMethodName());
+ TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
int numOfRegions = -1;
try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index 4c8c4ce..0446f61 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -125,7 +125,8 @@ public class TestRegionObserverScannerOpenHook {
Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s, long readPt)
throws IOException {
scan.setFilter(new NoDataFilter());
- return new StoreScanner((HStore) store, store.getScanInfo(), scan, targetCols, readPt);
+ HStore hs = (HStore) store;
+ return new StoreScanner(hs, hs.getScanInfo(), scan, targetCols, readPt);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
index ece658b..e94859a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
@@ -25,14 +25,15 @@ import java.util.Collection;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.testclassification.IOTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
import org.junit.Test;
@@ -109,7 +110,7 @@ public class TestForceCacheImportantBlocks {
setBloomFilterType(BLOOM_TYPE);
hcd.setBlocksize(BLOCK_SIZE);
hcd.setBlockCacheEnabled(cfCacheEnabled);
- Region region = TEST_UTIL.createTestRegion(TABLE, hcd);
+ HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd);
BlockCache cache = region.getStore(hcd.getName()).getCacheConfig().getBlockCache();
CacheStats stats = cache.getStats();
writeTestData(region);
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
index fae7247..6c7d686 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
@@ -30,21 +30,21 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.ChunkCreator;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl;
-import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -307,7 +307,7 @@ public class TestScannerFromBucketCache {
put.add(kv21);
region.put(put);
region.flush(true);
- Store store = region.getStore(fam1);
+ HStore store = region.getStore(fam1);
while (store.getStorefilesCount() <= 0) {
try {
Thread.sleep(20);
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DelegatingKeyValueScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DelegatingKeyValueScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DelegatingKeyValueScanner.java
index be28d17..da71c46 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DelegatingKeyValueScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DelegatingKeyValueScanner.java
@@ -69,7 +69,7 @@ public class DelegatingKeyValueScanner implements KeyValueScanner {
}
@Override
- public boolean shouldUseScanner(Scan scan, Store store, long oldestUnexpiredTS) {
+ public boolean shouldUseScanner(Scan scan, HStore store, long oldestUnexpiredTS) {
return delegate.shouldUseScanner(scan, store, oldestUnexpiredTS);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
index 2046816..bde28a2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -54,10 +54,11 @@ public class NoOpScanPolicyObserver implements RegionCoprocessor, RegionObserver
public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
Store store, List<KeyValueScanner> scanners, InternalScanner s, long readPoint)
throws IOException {
- ScanInfo oldSI = store.getScanInfo();
+ HStore hs = (HStore) store;
+ ScanInfo oldSI = hs.getScanInfo();
ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(),
oldSI.getTtl(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
- return new StoreScanner((HStore) store, scanInfo, OptionalInt.empty(), scanners,
+ return new StoreScanner(hs, scanInfo, OptionalInt.empty(), scanners,
ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
}
@@ -69,11 +70,12 @@ public class NoOpScanPolicyObserver implements RegionCoprocessor, RegionObserver
final ObserverContext<RegionCoprocessorEnvironment> c, Store store,
List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException {
+ HStore hs = (HStore) store;
// this demonstrates how to override the scanners default behavior
- ScanInfo oldSI = store.getScanInfo();
+ ScanInfo oldSI = hs.getScanInfo();
ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(),
oldSI.getTtl(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
- return new StoreScanner((HStore) store, scanInfo, OptionalInt.empty(), scanners, scanType,
+ return new StoreScanner(hs, scanInfo, OptionalInt.empty(), scanners, scanType,
store.getSmallestReadPoint(), earliestPutTs);
}
@@ -81,11 +83,12 @@ public class NoOpScanPolicyObserver implements RegionCoprocessor, RegionObserver
public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s, long readPoint)
throws IOException {
+ HStore hs = (HStore) store;
Region r = c.getEnvironment().getRegion();
return scan.isReversed()
- ? new ReversedStoreScanner((HStore) store, store.getScanInfo(), scan, targetCols,
+ ? new ReversedStoreScanner(hs, hs.getScanInfo(), scan, targetCols,
r.getReadPoint(scan.getIsolationLevel()))
- : new StoreScanner((HStore) store, store.getScanInfo(), scan, targetCols,
+ : new StoreScanner(hs, hs.getScanInfo(), scan, targetCols,
r.getReadPoint(scan.getIsolationLevel()));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 2427cad..6a7e98b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -17,6 +17,21 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
+import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -60,21 +75,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
-import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
/**
* Testing of HRegion.incrementColumnValue, HRegion.increment,
* and HRegion.append
@@ -84,7 +84,7 @@ public class TestAtomicOperation {
private static final Log LOG = LogFactory.getLog(TestAtomicOperation.class);
@Rule public TestName name = new TestName();
- Region region = null;
+ HRegion region = null;
private HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
// Test names
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
index 2d08e50..feb456e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -208,7 +208,7 @@ public class TestCacheOnWriteInSchema {
public void testCacheOnWriteInSchema() throws IOException {
// Write some random data into the store
StoreFileWriter writer = store.createWriterInTmp(Integer.MAX_VALUE,
- HFile.DEFAULT_COMPRESSION_ALGORITHM, false, true, false);
+ HFile.DEFAULT_COMPRESSION_ALGORITHM, false, true, false, false);
writeStoreFile(writer);
writer.close();
// Verify the block types of interest were cached on write
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 356054e..349815f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
+import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -182,7 +183,7 @@ public class TestCompaction {
spyR.compactStores();
// ensure that the compaction stopped, all old files are intact,
- Store s = r.stores.get(COLUMN_FAMILY);
+ HStore s = r.getStore(COLUMN_FAMILY);
assertEquals(compactionThreshold, s.getStorefilesCount());
assertTrue(s.getStorefilesSize() > 15*1000);
// and no new store files persisted past compactStores()
@@ -210,8 +211,7 @@ public class TestCompaction {
// Multiple versions allowed for an entry, so the delete isn't enough
// Lower TTL and expire to ensure that all our entries have been wiped
final int ttl = 1000;
- for (Store hstore: this.r.stores.values()) {
- HStore store = (HStore)hstore;
+ for (HStore store: this.r.stores.values()) {
ScanInfo old = store.getScanInfo();
ScanInfo si = new ScanInfo(old.getConfiguration(), old.getFamily(), old.getMinVersions(),
old.getMaxVersions(), ttl, old.getKeepDeletedCells(), HConstants.DEFAULT_BLOCKSIZE, 0,
@@ -307,7 +307,7 @@ public class TestCompaction {
CountDownLatch latch = new CountDownLatch(1);
Tracker tracker = new Tracker(latch);
- thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, tracker,
+ thread.requestCompaction(r, store, "test custom comapction", PRIORITY_USER, tracker,
null);
// wait for the latch to complete.
latch.await();
@@ -340,7 +340,7 @@ public class TestCompaction {
CountDownLatch latch = new CountDownLatch(1);
Tracker tracker = new Tracker(latch);
- thread.requestCompaction(mockRegion, store, "test custom comapction", Store.PRIORITY_USER,
+ thread.requestCompaction(mockRegion, store, "test custom comapction", PRIORITY_USER,
tracker, null);
// wait for the latch to complete.
latch.await(120, TimeUnit.SECONDS);
@@ -380,7 +380,7 @@ public class TestCompaction {
createStoreFile(r, store.getColumnFamilyName());
createStoreFile(r, store.getColumnFamilyName());
createStoreFile(r, store.getColumnFamilyName());
- thread.requestCompaction(r, store, "test mulitple custom comapctions", Store.PRIORITY_USER,
+ thread.requestCompaction(r, store, "test mulitple custom comapctions", PRIORITY_USER,
tracker, null);
}
// wait for the latch to complete.
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
index a3184b4..5735e88 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
@@ -180,7 +180,7 @@ public class TestCompactionFileNotFound {
// Refresh store files post compaction, this should not open already compacted files
hr1.refreshStoreFiles(true);
// Archive the store files and try another compaction to see if all is good
- for (Store store : hr1.getStores()) {
+ for (HStore store : hr1.getStores()) {
store.closeAndArchiveCompactedFiles();
}
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index 9c33d28..d8d38fa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -288,10 +288,9 @@ public class TestCompoundBloomFilter {
byte[] qualifier) {
Scan scan = new Scan().withStartRow(row).withStopRow(row, true);
scan.addColumn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME), qualifier);
- Store store = mock(Store.class);
- HColumnDescriptor hcd = mock(HColumnDescriptor.class);
- when(hcd.getName()).thenReturn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME));
- when(store.getColumnFamilyDescriptor()).thenReturn(hcd);
+ HStore store = mock(HStore.class);
+ when(store.getColumnFamilyDescriptor())
+ .thenReturn(ColumnFamilyDescriptorBuilder.of(RandomKeyValueUtil.COLUMN_FAMILY_NAME));
return scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
index fc0659f..066e686 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
+
import java.io.IOException;
import java.security.Key;
import java.security.SecureRandom;
@@ -539,7 +541,7 @@ public class TestHMobStore {
// Trigger major compaction
this.store.triggerMajorCompaction();
Optional<CompactionContext> requestCompaction =
- this.store.requestCompaction(Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
+ this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
Assert.assertEquals(1, this.store.getStorefiles().size());
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 0b0d651..9fcdf56 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -284,7 +284,7 @@ public class TestHRegion {
@Test
public void testCloseCarryingSnapshot() throws IOException {
HRegion region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
- Store store = region.getStore(COLUMN_FAMILY_BYTES);
+ HStore store = region.getStore(COLUMN_FAMILY_BYTES);
// Get some random bytes.
byte [] value = Bytes.toBytes(method);
// Make a random put against our cf.
@@ -333,7 +333,7 @@ public class TestHRegion {
HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, faultyLog,
COLUMN_FAMILY_BYTES);
- Store store = region.getStore(COLUMN_FAMILY_BYTES);
+ HStore store = region.getStore(COLUMN_FAMILY_BYTES);
// Get some random bytes.
byte [] value = Bytes.toBytes(method);
faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
@@ -350,7 +350,7 @@ public class TestHRegion {
} finally {
assertTrue("The regionserver should have thrown an exception", threwIOE);
}
- long sz = store.getSizeToFlush().getDataSize();
+ long sz = store.getFlushableSize().getDataSize();
assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
HBaseTestingUtility.closeRegionAndWAL(region);
}
@@ -382,7 +382,7 @@ public class TestHRegion {
FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog,
COLUMN_FAMILY_BYTES);
- Store store = region.getStore(COLUMN_FAMILY_BYTES);
+ HStore store = region.getStore(COLUMN_FAMILY_BYTES);
assertEquals(0, region.getMemstoreSize());
// Put some value and make sure flush could be completed normally
@@ -394,7 +394,7 @@ public class TestHRegion {
assertTrue(onePutSize > 0);
region.flush(true);
assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
- assertEquals("flushable size should be zero", 0, store.getSizeToFlush().getDataSize());
+ assertEquals("flushable size should be zero", 0, store.getFlushableSize().getDataSize());
// save normalCPHost and replaced by mockedCPHost, which will cancel flush requests
RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
@@ -406,13 +406,13 @@ public class TestHRegion {
region.flush(true);
assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
assertEquals("flushable size should NOT be zero", onePutSize,
- store.getSizeToFlush().getDataSize());
+ store.getFlushableSize().getDataSize());
// set normalCPHost and flush again, the snapshot will be flushed
region.setCoprocessorHost(normalCPHost);
region.flush(true);
assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
- assertEquals("flushable size should be zero", 0, store.getSizeToFlush().getDataSize());
+ assertEquals("flushable size should be zero", 0, store.getFlushableSize().getDataSize());
HBaseTestingUtility.closeRegionAndWAL(region);
}
@@ -424,7 +424,7 @@ public class TestHRegion {
FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog,
COLUMN_FAMILY_BYTES);
- Store store = region.getStore(COLUMN_FAMILY_BYTES);
+ HStore store = region.getStore(COLUMN_FAMILY_BYTES);
assertEquals(0, region.getMemstoreSize());
// Put one value
@@ -450,7 +450,7 @@ public class TestHRegion {
long expectedSize = onePutSize * 2;
assertEquals("memstoreSize should be incremented", expectedSize, region.getMemstoreSize());
assertEquals("flushable size should be incremented", expectedSize,
- store.getSizeToFlush().getDataSize());
+ store.getFlushableSize().getDataSize());
region.setCoprocessorHost(null);
HBaseTestingUtility.closeRegionAndWAL(region);
@@ -565,7 +565,7 @@ public class TestHRegion {
p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
region.put(p1);
// Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only.
- Store store = region.getStore(COLUMN_FAMILY_BYTES);
+ HStore store = region.getStore(COLUMN_FAMILY_BYTES);
StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
storeFlushCtx.prepare();
// Now add two entries to the foreground memstore.
@@ -699,7 +699,7 @@ public class TestHRegion {
}
MonitoredTask status = TaskMonitor.get().createStatus(method);
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
@@ -751,7 +751,7 @@ public class TestHRegion {
long recoverSeqId = 1030;
MonitoredTask status = TaskMonitor.get().createStatus(method);
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
@@ -796,7 +796,7 @@ public class TestHRegion {
dos.close();
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
@@ -854,7 +854,7 @@ public class TestHRegion {
long recoverSeqId = 1030;
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
MonitoredTask status = TaskMonitor.get().createStatus(method);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
@@ -3713,7 +3713,7 @@ public class TestHRegion {
if (i != 0 && i % compactInterval == 0) {
region.compact(true);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
store.closeAndArchiveCompactedFiles();
}
}
@@ -3893,7 +3893,7 @@ public class TestHRegion {
// Compact regularly to avoid creating too many files and exceeding
// the ulimit.
region.compact(false);
- for (Store store : region.getStores()) {
+ for (HStore store : region.getStores()) {
store.closeAndArchiveCompactedFiles();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index f13c509..63f5dfc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -35,8 +35,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@@ -53,34 +51,23 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
-import org.apache.hadoop.hbase.wal.WALEdit;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -89,6 +76,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay;
@@ -100,6 +88,19 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;
+
/**
* Tests of HRegion methods for replaying flush, compaction, region open, etc events for secondary
* region replicas
@@ -127,7 +128,7 @@ public class TestHRegionReplayEvents {
// per test fields
private Path rootDir;
- private HTableDescriptor htd;
+ private TableDescriptor htd;
private long time;
private RegionServerServices rss;
private HRegionInfo primaryHri, secondaryHri;
@@ -146,11 +147,11 @@ public class TestHRegionReplayEvents {
rootDir = new Path(dir + method);
TEST_UTIL.getConfiguration().set(HConstants.HBASE_DIR, rootDir.toString());
method = name.getMethodName();
-
- htd = new HTableDescriptor(TableName.valueOf(method));
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(method));
for (byte[] family : families) {
- htd.addFamily(new HColumnDescriptor(family));
+ builder.addColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
}
+ htd = builder.build();
time = System.currentTimeMillis();
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
@@ -338,10 +339,10 @@ public class TestHRegionReplayEvents {
if (flushDesc != null) {
// first verify that everything is replayed and visible before flush event replay
verifyData(secondaryRegion, 0, lastReplayed, cq, families);
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long storeMemstoreSize = store.getMemStoreSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long storeMemstoreSize = store.getMemStoreSize().getHeapSize();
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
- long storeFlushableSize = store.getFlushableSize();
+ long storeFlushableSize = store.getFlushableSize().getHeapSize();
long storeSize = store.getSize();
long storeSizeUncompressed = store.getStoreSizeUncompressed();
if (flushDesc.getAction() == FlushAction.START_FLUSH) {
@@ -351,7 +352,7 @@ public class TestHRegionReplayEvents {
assertEquals(result.flushOpSeqId, flushDesc.getFlushSequenceNumber());
// assert that the store memstore is smaller now
- long newStoreMemstoreSize = store.getMemStoreSize();
+ long newStoreMemstoreSize = store.getMemStoreSize().getHeapSize();
LOG.info("Memstore size reduced by:"
+ StringUtils.humanReadableInt(newStoreMemstoreSize - storeMemstoreSize));
assertTrue(storeMemstoreSize > newStoreMemstoreSize);
@@ -362,10 +363,10 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount++;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- long newFlushableSize = store.getFlushableSize();
+ long newFlushableSize = store.getFlushableSize().getHeapSize();
assertTrue(storeFlushableSize > newFlushableSize);
// assert that the region memstore is smaller now
@@ -383,7 +384,7 @@ public class TestHRegionReplayEvents {
secondaryRegion.replayWALCompactionMarker(compactionDesc, true, false, Long.MAX_VALUE);
// assert that the compaction is applied
- for (Store store : secondaryRegion.getStores()) {
+ for (HStore store : secondaryRegion.getStores()) {
if (store.getColumnFamilyName().equals("cf1")) {
assertEquals(1, store.getStorefilesCount());
} else {
@@ -401,7 +402,7 @@ public class TestHRegionReplayEvents {
LOG.info("-- Verifying edits from primary. Ensuring that files are not deleted");
verifyData(primaryRegion, 0, lastReplayed, cq, families);
- for (Store store : primaryRegion.getStores()) {
+ for (HStore store : primaryRegion.getStores()) {
if (store.getColumnFamilyName().equals("cf1")) {
assertEquals(1, store.getStorefilesCount());
} else {
@@ -437,10 +438,10 @@ public class TestHRegionReplayEvents {
= WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
if (flushDesc != null) {
// first verify that everything is replayed and visible before flush event replay
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long storeMemstoreSize = store.getMemStoreSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long storeMemstoreSize = store.getMemStoreSize().getHeapSize();
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
- long storeFlushableSize = store.getFlushableSize();
+ long storeFlushableSize = store.getFlushableSize().getHeapSize();
if (flushDesc.getAction() == FlushAction.START_FLUSH) {
startFlushDesc = flushDesc;
@@ -452,7 +453,7 @@ public class TestHRegionReplayEvents {
assertTrue(storeFlushableSize > 0);
// assert that the store memstore is smaller now
- long newStoreMemstoreSize = store.getMemStoreSize();
+ long newStoreMemstoreSize = store.getMemStoreSize().getHeapSize();
LOG.info("Memstore size reduced by:"
+ StringUtils.humanReadableInt(newStoreMemstoreSize - storeMemstoreSize));
assertTrue(storeMemstoreSize > newStoreMemstoreSize);
@@ -571,7 +572,7 @@ public class TestHRegionReplayEvents {
// no store files in the region
int expectedStoreFileCount = 0;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
@@ -586,11 +587,11 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount++;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long newFlushableSize = store.getFlushableSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long newFlushableSize = store.getFlushableSize().getHeapSize();
assertTrue(newFlushableSize > 0); // assert that the memstore is not dropped
// assert that the region memstore is same as before
@@ -661,7 +662,7 @@ public class TestHRegionReplayEvents {
// no store files in the region
int expectedStoreFileCount = 0;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
@@ -676,11 +677,11 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount++;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long newFlushableSize = store.getFlushableSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long newFlushableSize = store.getFlushableSize().getHeapSize();
assertTrue(newFlushableSize > 0); // assert that the memstore is not dropped
// assert that the region memstore is smaller than before, but not empty
@@ -762,7 +763,7 @@ public class TestHRegionReplayEvents {
// no store files in the region
int expectedStoreFileCount = 0;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
@@ -772,8 +773,8 @@ public class TestHRegionReplayEvents {
assertTrue(commitFlushDesc.getFlushSequenceNumber() > 0);
// ensure all files are visible in secondary
- for (Store store : secondaryRegion.getStores()) {
- assertTrue(store.getMaxSequenceId() <= secondaryRegion.getReadPoint(null));
+ for (HStore store : secondaryRegion.getStores()) {
+ assertTrue(store.getMaxSequenceId().orElse(0L) <= secondaryRegion.getReadPoint(null));
}
LOG.info("-- Replaying flush commit in secondary" + commitFlushDesc);
@@ -781,11 +782,11 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount++;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long newFlushableSize = store.getFlushableSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long newFlushableSize = store.getFlushableSize().getHeapSize();
if (droppableMemstore) {
// assert that the memstore is dropped
assertTrue(newFlushableSize == MutableSegment.DEEP_OVERHEAD);
@@ -861,7 +862,7 @@ public class TestHRegionReplayEvents {
// no store files in the region
int expectedStoreFileCount = 0;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
long regionMemstoreSize = secondaryRegion.getMemstoreSize();
@@ -873,11 +874,11 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount++;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- long newFlushableSize = store.getFlushableSize();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ long newFlushableSize = store.getFlushableSize().getHeapSize();
assertTrue(newFlushableSize == MutableSegment.DEEP_OVERHEAD);
// assert that the region memstore is empty
@@ -942,7 +943,7 @@ public class TestHRegionReplayEvents {
// no store files in the region
int expectedStoreFileCount = 0;
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
@@ -952,11 +953,11 @@ public class TestHRegionReplayEvents {
// assert that the flush files are picked
expectedStoreFileCount = 2; // two flushes happened
- for (Store s : secondaryRegion.getStores()) {
+ for (HStore s : secondaryRegion.getStores()) {
assertEquals(expectedStoreFileCount, s.getStorefilesCount());
}
- Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
- MemstoreSize newSnapshotSize = store.getSizeOfSnapshot();
+ HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+ MemstoreSize newSnapshotSize = store.getSnapshotSize();
assertTrue(newSnapshotSize.getDataSize() == 0);
// assert that the region memstore is empty
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
index b685115..815166b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,18 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.ref.SoftReference;
@@ -60,19 +59,22 @@ import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MemoryCompactionPolicy;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -95,10 +97,12 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -109,7 +113,7 @@ import org.mockito.Mockito;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
/**
- * Test class for the Store
+ * Test class for the HStore
*/
@Category({ RegionServerTests.class, MediumTests.class })
public class TestHStore {
@@ -117,6 +121,7 @@ public class TestHStore {
@Rule
public TestName name = new TestName();
+ HRegion region;
HStore store;
byte [] table = Bytes.toBytes("table");
byte [] family = Bytes.toBytes("family");
@@ -138,8 +143,8 @@ public class TestHStore {
long id = System.currentTimeMillis();
Get get = new Get(row);
- private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
/**
@@ -164,55 +169,51 @@ public class TestHStore {
init(methodName, TEST_UTIL.getConfiguration());
}
- private Store init(String methodName, Configuration conf) throws IOException {
- HColumnDescriptor hcd = new HColumnDescriptor(family);
+ private HStore init(String methodName, Configuration conf) throws IOException {
// some of the tests write 4 versions and then flush
// (with HBASE-4241, lower versions are collected on flush)
- hcd.setMaxVersions(4);
- return init(methodName, conf, hcd);
+ return init(methodName, conf,
+ ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build());
}
- private HStore init(String methodName, Configuration conf, HColumnDescriptor hcd)
+ private HStore init(String methodName, Configuration conf, ColumnFamilyDescriptor hcd)
throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
- return init(methodName, conf, htd, hcd);
+ return init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd);
}
- private HStore init(String methodName, Configuration conf, HTableDescriptor htd,
- HColumnDescriptor hcd) throws IOException {
- return init(methodName, conf, htd, hcd, null);
+ private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
+ ColumnFamilyDescriptor hcd) throws IOException {
+ return init(methodName, conf, builder, hcd, null);
}
- @SuppressWarnings("deprecation")
- private HStore init(String methodName, Configuration conf, HTableDescriptor htd,
- HColumnDescriptor hcd, MyStoreHook hook) throws IOException {
- return init(methodName, conf, htd, hcd, hook, false);
+ private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
+ ColumnFamilyDescriptor hcd, MyStoreHook hook) throws IOException {
+ return init(methodName, conf, builder, hcd, hook, false);
}
- @SuppressWarnings("deprecation")
- private HStore init(String methodName, Configuration conf, HTableDescriptor htd,
- HColumnDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
- //Setting up a Store
- Path basedir = new Path(DIR+methodName);
+
+ private void initHRegion(String methodName, Configuration conf, TableDescriptorBuilder builder,
+ ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
+ TableDescriptor htd = builder.addColumnFamily(hcd).build();
+ Path basedir = new Path(DIR + methodName);
Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));
FileSystem fs = FileSystem.get(conf);
fs.delete(logdir, true);
-
- if (htd.hasFamily(hcd.getName())) {
- htd.modifyFamily(hcd);
- } else {
- htd.addFamily(hcd);
- }
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- final Configuration walConf = new Configuration(conf);
+ Configuration walConf = new Configuration(conf);
FSUtils.setRootDir(walConf, basedir);
- final WALFactory wals = new WALFactory(walConf, null, methodName);
- HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(),
- info.getTable().getNamespace()), fs, conf, info, htd, null);
+ WALFactory wals = new WALFactory(walConf, null, methodName);
+ region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info),
+ wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), conf, htd, null);
+ }
+
+ private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
+ ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
+ initHRegion(methodName, conf, builder, hcd, hook, switchToPread);
if (hook == null) {
store = new HStore(region, hcd, conf);
} else {
@@ -299,13 +300,14 @@ public class TestHStore {
Configuration conf = HBaseConfiguration.create();
FileSystem fs = FileSystem.get(conf);
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setCompressionType(Compression.Algorithm.GZ);
- hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
+ ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
+ .setCompressionType(Compression.Algorithm.GZ).setDataBlockEncoding(DataBlockEncoding.DIFF)
+ .build();
init(name.getMethodName(), conf, hcd);
// Test createWriterInTmp()
- StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false);
+ StoreFileWriter writer =
+ store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false);
Path path = writer.getPath();
writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
@@ -341,10 +343,8 @@ public class TestHStore {
// Set the compaction threshold higher to avoid normal compactions.
conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5);
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setMinVersions(minVersions);
- hcd.setTimeToLive(ttl);
- init(name.getMethodName() + "-" + minVersions, conf, hcd);
+ init(name.getMethodName() + "-" + minVersions, conf, ColumnFamilyDescriptorBuilder
+ .newBuilder(family).setMinVersions(minVersions).setTimeToLive(ttl).build());
long storeTtl = this.store.getScanInfo().getTtl();
long sleepTime = storeTtl / storeFileNum;
@@ -605,6 +605,22 @@ public class TestHStore {
@After
public void tearDown() throws Exception {
EnvironmentEdgeManagerTestHelper.reset();
+ if (store != null) {
+ try {
+ store.close();
+ } catch (IOException e) {
+ }
+ store = null;
+ }
+ if (region != null) {
+ region.close();
+ region = null;
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws IOException {
+ TEST_UTIL.cleanupTestDir();
}
@Test
@@ -830,17 +846,19 @@ public class TestHStore {
// HTD overrides XML.
--anyValue;
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
- init(name.getMethodName() + "-htd", conf, htd, hcd);
+ init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder
+ .newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)),
+ ColumnFamilyDescriptorBuilder.of(family));
assertTrue(store.throttleCompaction(anyValue + 1));
assertFalse(store.throttleCompaction(anyValue));
// HCD overrides them both.
--anyValue;
- hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
- init(name.getMethodName() + "-hcd", conf, htd, hcd);
+ init(name.getMethodName() + "-hcd", conf,
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY,
+ Long.toString(anyValue)),
+ ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue))
+ .build());
assertTrue(store.throttleCompaction(anyValue + 1));
assertFalse(store.throttleCompaction(anyValue));
}
@@ -868,7 +886,7 @@ public class TestHStore {
private void addStoreFile() throws IOException {
HStoreFile f = this.store.getStorefiles().iterator().next();
Path storedir = f.getPath().getParent();
- long seqid = this.store.getMaxSequenceId();
+ long seqid = this.store.getMaxSequenceId().orElse(0L);
Configuration c = TEST_UTIL.getConfiguration();
FileSystem fs = FileSystem.get(c);
HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
@@ -995,20 +1013,23 @@ public class TestHStore {
public void testNumberOfMemStoreScannersAfterFlush() throws IOException {
long seqId = 100;
long timestamp = System.currentTimeMillis();
- Cell cell0 = CellUtil.createCell(row, family, qf1, timestamp,
- KeyValue.Type.Put.getCode(), qf1);
+ Cell cell0 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
+ .setQualifier(qf1).setTimestamp(timestamp).setType(KeyValue.Type.Put.getCode())
+ .setValue(qf1).build();
CellUtil.setSequenceId(cell0, seqId);
- testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Collections.EMPTY_LIST);
+ testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Collections.emptyList());
- Cell cell1 = CellUtil.createCell(row, family, qf2, timestamp,
- KeyValue.Type.Put.getCode(), qf1);
+ Cell cell1 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
+ .setQualifier(qf2).setTimestamp(timestamp).setType(KeyValue.Type.Put.getCode())
+ .setValue(qf1).build();
CellUtil.setSequenceId(cell1, seqId);
testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1));
seqId = 101;
timestamp = System.currentTimeMillis();
- Cell cell2 = CellUtil.createCell(row2, family, qf2, timestamp,
- KeyValue.Type.Put.getCode(), qf1);
+ Cell cell2 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row2).setFamily(family)
+ .setQualifier(qf2).setTimestamp(timestamp).setType(KeyValue.Type.Put.getCode())
+ .setValue(qf1).build();
CellUtil.setSequenceId(cell2, seqId);
testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1, cell2));
}
@@ -1052,15 +1073,16 @@ public class TestHStore {
}
}
- private Cell createCell(byte[] qualifier, long ts, long sequenceId, byte[] value) throws IOException {
- Cell c = CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put.getCode(), value);
- CellUtil.setSequenceId(c, sequenceId);
- return c;
+ private Cell createCell(byte[] qualifier, long ts, long sequenceId, byte[] value)
+ throws IOException {
+ return createCell(row, qualifier, ts, sequenceId, value);
}
private Cell createCell(byte[] row, byte[] qualifier, long ts, long sequenceId, byte[] value)
throws IOException {
- Cell c = CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put.getCode(), value);
+ Cell c = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
+ .setQualifier(qualifier).setTimestamp(ts).setType(KeyValue.Type.Put.getCode())
+ .setValue(value).build();
CellUtil.setSequenceId(c, sequenceId);
return c;
}
@@ -1154,8 +1176,6 @@ public class TestHStore {
private void testFlushBeforeCompletingScan(MyListHook hook, Filter filter, int expectedSize)
throws IOException, InterruptedException {
Configuration conf = HBaseConfiguration.create();
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setMaxVersions(1);
byte[] r0 = Bytes.toBytes("row0");
byte[] r1 = Bytes.toBytes("row1");
byte[] r2 = Bytes.toBytes("row2");
@@ -1165,12 +1185,14 @@ public class TestHStore {
MemstoreSize memStoreSize = new MemstoreSize();
long ts = EnvironmentEdgeManager.currentTime();
long seqId = 100;
- init(name.getMethodName(), conf, new HTableDescriptor(TableName.valueOf(table)), hcd, new MyStoreHook() {
- @Override
- public long getSmallestReadPoint(HStore store) {
- return seqId + 3;
- }
- });
+ init(name.getMethodName(), conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
+ ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(1).build(),
+ new MyStoreHook() {
+ @Override
+ public long getSmallestReadPoint(HStore store) {
+ return seqId + 3;
+ }
+ });
// The cells having the value0 won't be flushed to disk because the value of max version is 1
store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSize);
store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSize);
@@ -1216,9 +1238,8 @@ public class TestHStore {
public void testCreateScannerAndSnapshotConcurrently() throws IOException, InterruptedException {
Configuration conf = HBaseConfiguration.create();
conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore.class.getName());
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setInMemoryCompaction(MemoryCompactionPolicy.BASIC);
- init(name.getMethodName(), conf, hcd);
+ init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
+ .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
byte[] value = Bytes.toBytes("value");
MemstoreSize memStoreSize = new MemstoreSize();
long ts = EnvironmentEdgeManager.currentTime();
@@ -1408,9 +1429,8 @@ public class TestHStore {
conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushSize));
// Set the lower threshold to invoke the "MERGE" policy
conf.set(MemStoreCompactor.COMPACTING_MEMSTORE_THRESHOLD_KEY, String.valueOf(0));
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setInMemoryCompaction(MemoryCompactionPolicy.BASIC);
- init(name.getMethodName(), conf, hcd);
+ init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
+ .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
byte[] value = Bytes.toBytes("thisisavarylargevalue");
MemstoreSize memStoreSize = new MemstoreSize();
long ts = EnvironmentEdgeManager.currentTime();
@@ -1445,18 +1465,57 @@ public class TestHStore {
storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
}
+ @Test
+ public void testAge() throws IOException {
+ long currentTime = System.currentTimeMillis();
+ ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
+ edge.setValue(currentTime);
+ EnvironmentEdgeManager.injectEdge(edge);
+ Configuration conf = TEST_UTIL.getConfiguration();
+ ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(family);
+ initHRegion(name.getMethodName(), conf,
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd, null, false);
+ HStore store = new HStore(region, hcd, conf) {
+
+ @Override
+ protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf,
+ CellComparator kvComparator) throws IOException {
+ List<HStoreFile> storefiles =
+ Arrays.asList(mockStoreFile(currentTime - 10), mockStoreFile(currentTime - 100),
+ mockStoreFile(currentTime - 1000), mockStoreFile(currentTime - 10000));
+ StoreFileManager sfm = mock(StoreFileManager.class);
+ when(sfm.getStorefiles()).thenReturn(storefiles);
+ StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class);
+ when(storeEngine.getStoreFileManager()).thenReturn(sfm);
+ return storeEngine;
+ }
+ };
+ assertEquals(10L, store.getMinStoreFileAge().getAsLong());
+ assertEquals(10000L, store.getMaxStoreFileAge().getAsLong());
+ assertEquals((10 + 100 + 1000 + 10000) / 4.0, store.getAvgStoreFileAge().getAsDouble(), 1E-4);
+ }
+
+ private HStoreFile mockStoreFile(long createdTime) {
+ StoreFileInfo info = mock(StoreFileInfo.class);
+ when(info.getCreatedTimestamp()).thenReturn(createdTime);
+ HStoreFile sf = mock(HStoreFile.class);
+ when(sf.getReader()).thenReturn(mock(StoreFileReader.class));
+ when(sf.isHFile()).thenReturn(true);
+ when(sf.getFileInfo()).thenReturn(info);
+ return sf;
+ }
+
private MyStore initMyStore(String methodName, Configuration conf, MyStoreHook hook)
throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setMaxVersions(5);
- return (MyStore) init(methodName, conf, htd, hcd, hook);
+ return (MyStore) init(methodName, conf,
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
+ ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook);
}
- class MyStore extends HStore {
+ private class MyStore extends HStore {
private final MyStoreHook hook;
- MyStore(final HRegion region, final HColumnDescriptor family, final Configuration confParam,
+ MyStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam,
MyStoreHook hook, boolean switchToPread) throws IOException {
super(region, family, confParam);
this.hook = hook;
@@ -1479,8 +1538,10 @@ public class TestHStore {
}
private abstract class MyStoreHook {
+
void getScanners(MyStore store) throws IOException {
}
+
long getSmallestReadPoint(HStore store) {
return store.getHRegion().getSmallestReadPoint();
}
@@ -1488,13 +1549,10 @@ public class TestHStore {
@Test
public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws Exception {
- int flushSize = 500;
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.hstore.engine.class", DummyStoreEngine.class.getName());
conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 0);
// Set the lower threshold to invoke the "MERGE" policy
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setInMemoryCompaction(MemoryCompactionPolicy.BASIC);
MyStore store = initMyStore(name.getMethodName(), conf, new MyStoreHook() {});
MemstoreSize memStoreSize = new MemstoreSize();
long ts = System.currentTimeMillis();
@@ -1520,7 +1578,6 @@ public class TestHStore {
flushStore(store, seqID);
assertEquals(3, store.getStorefilesCount());
- ScanInfo scanInfo = store.getScanInfo();
Scan scan = new Scan();
scan.addFamily(family);
Collection<HStoreFile> storefiles2 = store.getStorefiles();
@@ -1547,7 +1604,6 @@ public class TestHStore {
ArrayList<HStoreFile> actualStorefiles1 = Lists.newArrayList(storefiles2);
actualStorefiles1.removeAll(actualStorefiles);
// Do compaction
- List<Exception> exceptions = new ArrayList<Exception>();
MyThread thread = new MyThread(storeScanner);
thread.start();
store.replaceStoreFiles(actualStorefiles, actualStorefiles1);
@@ -1684,7 +1740,7 @@ public class TestHStore {
public Object[] toArray() {return delegatee.toArray();}
@Override
- public <T> T[] toArray(T[] a) {return delegatee.toArray(a);}
+ public <R> R[] toArray(R[] a) {return delegatee.toArray(a);}
@Override
public boolean add(T e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f4c3b35/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
index b20cae8..2123a8a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -205,11 +207,10 @@ public class TestHStoreFile extends HBaseTestCase {
@Test
public void testEmptyStoreFileRestrictKeyRanges() throws Exception {
StoreFileReader reader = mock(StoreFileReader.class);
- Store store = mock(Store.class);
- HColumnDescriptor hcd = mock(HColumnDescriptor.class);
+ HStore store = mock(HStore.class);
byte[] cf = Bytes.toBytes("ty");
- when(hcd.getName()).thenReturn(cf);
- when(store.getColumnFamilyDescriptor()).thenReturn(hcd);
+ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(cf);
+ when(store.getColumnFamilyDescriptor()).thenReturn(cfd);
StoreFileScanner scanner =
new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0, 0, true);
Scan scan = new Scan();
@@ -526,10 +527,8 @@ public class TestHStoreFile extends HBaseTestCase {
Scan scan = new Scan(row.getBytes(),row.getBytes());
scan.addColumn("family".getBytes(), "family:col".getBytes());
- Store store = mock(Store.class);
- HColumnDescriptor hcd = mock(HColumnDescriptor.class);
- when(hcd.getName()).thenReturn(Bytes.toBytes("family"));
- when(store.getColumnFamilyDescriptor()).thenReturn(hcd);
+ HStore store = mock(HStore.class);
+ when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
if (i % 2 == 0) {
if (!exists) falseNeg++;
@@ -713,10 +712,8 @@ public class TestHStoreFile extends HBaseTestCase {
StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
- Store store = mock(Store.class);
- HColumnDescriptor hcd = mock(HColumnDescriptor.class);
- when(hcd.getName()).thenReturn(Bytes.toBytes("family"));
- when(store.getColumnFamilyDescriptor()).thenReturn(hcd);
+ HStore store = mock(HStore.class);
+ when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
// check false positives rate
int falsePos = 0;
int falseNeg = 0;
@@ -857,10 +854,8 @@ public class TestHStoreFile extends HBaseTestCase {
HStoreFile hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf,
BloomType.NONE, true);
- Store store = mock(Store.class);
- HColumnDescriptor hcd = mock(HColumnDescriptor.class);
- when(hcd.getName()).thenReturn(family);
- when(store.getColumnFamilyDescriptor()).thenReturn(hcd);
+ HStore store = mock(HStore.class);
+ when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of(family));
hsf.initReader();
StoreFileReader reader = hsf.getReader();
StoreFileScanner scanner = getStoreFileScanner(reader, false, false);