You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/11/13 07:54:17 UTC
svn commit: r1408621 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/io/hfile/slab/
test/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/regionserver/
Author: larsh
Date: Tue Nov 13 06:54:16 2012
New Revision: 1408621
URL: http://svn.apache.org/viewvc?rev=1408621&view=rev
Log:
HBASE-5898 Consider double-checked locking for block cache lock (Todd, Elliot, LarsH)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java Tue Nov 13 06:54:16 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.io.hfile
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
@@ -90,10 +89,6 @@ public abstract class AbstractHFileReade
/** Block cache configuration. */
protected final CacheConfig cacheConf;
- protected AtomicLong cacheHits = new AtomicLong();
- protected AtomicLong blockLoads = new AtomicLong();
- protected AtomicLong metaLoads = new AtomicLong();
-
/** Path of file */
protected final Path path;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java Tue Nov 13 06:54:16 2012
@@ -48,9 +48,12 @@ public interface BlockCache {
* Fetch block from cache.
* @param cacheKey Block to fetch.
* @param caching Whether this request has caching enabled (used for stats)
+ * @param repeat Whether this is a repeat lookup for the same block
+ * (used to avoid double counting cache misses when doing double-check locking)
+ * {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
* @return Block or null if block is not in 2 cache.
*/
- public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching);
+ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat);
/**
* Evict block from cache.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java Tue Nov 13 06:54:16 2012
@@ -90,14 +90,14 @@ public class DoubleBlockCache implements
}
@Override
- public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
Cacheable cachedBlock;
- if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching)) != null) {
+ if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
stats.hit(caching);
return cachedBlock;
- } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching)) != null) {
+ } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
if (caching) {
onHeapCache.cacheBlock(cacheKey, cachedBlock);
}
@@ -105,7 +105,7 @@ public class DoubleBlockCache implements
return cachedBlock;
}
- stats.miss(caching);
+ if (!repeat) stats.miss(caching);
return null;
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Tue Nov 13 06:54:16 2012
@@ -225,15 +225,12 @@ public class HFileReaderV1 extends Abstr
// Per meta key from any given file, synchronize reads for said block
synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
- metaLoads.incrementAndGet();
-
// Check cache for block. If found return.
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
(HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
- cacheConf.shouldCacheBlockOnRead(effectiveCategory));
+ cacheConf.shouldCacheBlockOnRead(effectiveCategory), false);
if (cachedBlock != null) {
- cacheHits.incrementAndGet();
getSchemaMetrics().updateOnCacheHit(effectiveCategory,
SchemaMetrics.NO_COMPACTION);
return cachedBlock.getBufferWithoutHeader();
@@ -290,15 +287,12 @@ public class HFileReaderV1 extends Abstr
// the other choice is to duplicate work (which the cache would prevent you
// from doing).
synchronized (dataBlockIndexReader.getRootBlockKey(block)) {
- blockLoads.incrementAndGet();
-
// Check cache for block. If found return.
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
(HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
- cacheConf.shouldCacheDataOnRead());
+ cacheConf.shouldCacheDataOnRead(), false);
if (cachedBlock != null) {
- cacheHits.incrementAndGet();
getSchemaMetrics().updateOnCacheHit(
cachedBlock.getBlockType().getCategory(), isCompaction);
return cachedBlock.getBufferWithoutHeader();
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Nov 13 06:54:16 2012
@@ -210,8 +210,6 @@ public class HFileReaderV2 extends Abstr
// is OK to do for meta blocks because the meta block index is always
// single-level.
synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
- metaLoads.incrementAndGet();
-
// Check cache for block. If found return.
long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
@@ -220,11 +218,10 @@ public class HFileReaderV2 extends Abstr
cacheBlock &= cacheConf.shouldCacheDataOnRead();
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
- (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
+ (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false);
if (cachedBlock != null) {
// Return a distinct 'shallow copy' of the block,
// so pos does not get messed by the scanner
- cacheHits.incrementAndGet();
getSchemaMetrics().updateOnCacheHit(BlockCategory.META, false);
return cachedBlock.getBufferWithoutHeader();
}
@@ -288,69 +285,85 @@ public class HFileReaderV2 extends Abstr
new BlockCacheKey(name, dataBlockOffset,
dataBlockEncoder.getEffectiveEncodingInCache(isCompaction),
expectedBlockType);
- IdLock.Entry lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+
+ boolean useLock = false;
+ IdLock.Entry lockEntry = null;
+
try {
- blockLoads.incrementAndGet();
+ while (true) {
- // Check cache for block. If found return.
- if (cacheConf.isBlockCacheEnabled()) {
- HFileBlock cachedBlock = (HFileBlock)
- cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
- if (cachedBlock != null) {
- BlockCategory blockCategory =
- cachedBlock.getBlockType().getCategory();
- cacheHits.incrementAndGet();
+ if (useLock) {
+ lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+ }
- getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
+ // Check cache for block. If found return.
+ if (cacheConf.isBlockCacheEnabled()) {
+ // Try and get the block from the block cache. If the useLock variable is true then this
+ // is the second time through the loop and it should not be counted as a block cache miss.
+ HFileBlock cachedBlock = (HFileBlock)
+ cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, useLock);
+ if (cachedBlock != null) {
+ BlockCategory blockCategory =
+ cachedBlock.getBlockType().getCategory();
- if (cachedBlock.getBlockType() == BlockType.DATA) {
- HFile.dataBlockReadCnt.incrementAndGet();
- }
+ getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
- validateBlockType(cachedBlock, expectedBlockType);
+ if (cachedBlock.getBlockType() == BlockType.DATA) {
+ HFile.dataBlockReadCnt.incrementAndGet();
+ }
- // Validate encoding type for encoded blocks. We include encoding
- // type in the cache key, and we expect it to match on a cache hit.
- if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
- cachedBlock.getDataBlockEncoding() !=
- dataBlockEncoder.getEncodingInCache()) {
- throw new IOException("Cached block under key " + cacheKey + " " +
- "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
- " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
+ validateBlockType(cachedBlock, expectedBlockType);
+
+ // Validate encoding type for encoded blocks. We include encoding
+ // type in the cache key, and we expect it to match on a cache hit.
+ if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
+ cachedBlock.getDataBlockEncoding() !=
+ dataBlockEncoder.getEncodingInCache()) {
+ throw new IOException("Cached block under key " + cacheKey + " " +
+ "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
+ " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
+ }
+ return cachedBlock;
}
- return cachedBlock;
+ // Carry on, please load.
+ }
+ if (!useLock) {
+ // check cache again with lock
+ useLock = true;
+ continue;
}
- // Carry on, please load.
- }
- // Load block from filesystem.
- long startTimeNs = System.nanoTime();
- HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
- onDiskBlockSize, -1, pread);
- hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock,
- isCompaction);
- validateBlockType(hfileBlock, expectedBlockType);
- passSchemaMetricsTo(hfileBlock);
- BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
+ // Load block from filesystem.
+ long startTimeNs = System.nanoTime();
+ HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
+ onDiskBlockSize, -1, pread);
+ hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock,
+ isCompaction);
+ validateBlockType(hfileBlock, expectedBlockType);
+ passSchemaMetricsTo(hfileBlock);
+ BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
- final long delta = System.nanoTime() - startTimeNs;
- HFile.offerReadLatency(delta, pread);
- getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
+ final long delta = System.nanoTime() - startTimeNs;
+ HFile.offerReadLatency(delta, pread);
+ getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
- // Cache the block if necessary
- if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
- hfileBlock.getBlockType().getCategory())) {
- cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
- cacheConf.isInMemory());
- }
+ // Cache the block if necessary
+ if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+ hfileBlock.getBlockType().getCategory())) {
+ cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
+ cacheConf.isInMemory());
+ }
- if (hfileBlock.getBlockType() == BlockType.DATA) {
- HFile.dataBlockReadCnt.incrementAndGet();
- }
+ if (hfileBlock.getBlockType() == BlockType.DATA) {
+ HFile.dataBlockReadCnt.incrementAndGet();
+ }
- return hfileBlock;
+ return hfileBlock;
+ }
} finally {
- offsetLock.releaseLockEntry(lockEntry);
+ if (lockEntry != null) {
+ offsetLock.releaseLockEntry(lockEntry);
+ }
}
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Nov 13 06:54:16 2012
@@ -327,13 +327,16 @@ public class LruBlockCache implements Bl
* Get the buffer of the block with the specified name.
* @param cacheKey block's cache key
* @param caching true if the caller caches blocks on cache misses
+ * @param repeat Whether this is a repeat lookup for the same block
+ * (used to avoid double counting cache misses when doing double-check locking)
+ * {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
* @return buffer of specified cache key, or null if not in cache
*/
@Override
- public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
CachedBlock cb = map.get(cacheKey);
if(cb == null) {
- stats.miss(caching);
+ if (!repeat) stats.miss(caching);
return null;
}
stats.hit(caching);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java Tue Nov 13 06:54:16 2012
@@ -68,7 +68,7 @@ public class SimpleBlockCache implements
return cache.size();
}
- public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+ public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
processQueue(); // clear out some crap.
Ref ref = cache.get(cacheKey);
if (ref == null)
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java Tue Nov 13 06:54:16 2012
@@ -152,10 +152,10 @@ public class SingleSizeCache implements
}
@Override
- public Cacheable getBlock(BlockCacheKey key, boolean caching) {
+ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
CacheablePair contentBlock = backingMap.get(key);
if (contentBlock == null) {
- stats.miss(caching);
+ if (!repeat) stats.miss(caching);
return null;
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java Tue Nov 13 06:54:16 2012
@@ -228,24 +228,25 @@ public class SlabCache implements SlabIt
/**
* Get the buffer of the block with the specified name.
- *
- * @param key
* @param caching
+ * @param key
+ * @param repeat
+ *
* @return buffer of specified block name, or null if not in cache
*/
- public Cacheable getBlock(BlockCacheKey key, boolean caching) {
+ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
SingleSizeCache cachedBlock = backingStore.get(key);
if (cachedBlock == null) {
- stats.miss(caching);
+ if (!repeat) stats.miss(caching);
return null;
}
- Cacheable contentBlock = cachedBlock.getBlock(key, caching);
+ Cacheable contentBlock = cachedBlock.getBlock(key, caching, false);
if (contentBlock != null) {
stats.hit(caching);
} else {
- stats.miss(caching);
+ if (!repeat) stats.miss(caching);
}
return contentBlock;
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java Tue Nov 13 06:54:16 2012
@@ -92,12 +92,12 @@ public class CacheTestUtils {
}
toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName,
- false);
+ false, false);
if (retrievedBlock != null) {
assertEquals(ourBlock.block, retrievedBlock);
toBeTested.evictBlock(ourBlock.blockName);
hits.incrementAndGet();
- assertNull(toBeTested.getBlock(ourBlock.blockName, false));
+ assertNull(toBeTested.getBlock(ourBlock.blockName, false, false));
} else {
miss.incrementAndGet();
}
@@ -125,7 +125,7 @@ public class CacheTestUtils {
HFileBlockPair[] blocks = generateHFileBlocks(numBlocks, blockSize);
// Confirm empty
for (HFileBlockPair block : blocks) {
- assertNull(toBeTested.getBlock(block.blockName, true));
+ assertNull(toBeTested.getBlock(block.blockName, true, false));
}
// Add blocks
@@ -138,7 +138,7 @@ public class CacheTestUtils {
// MapMaker makes no guarantees when it will evict, so neither can we.
for (HFileBlockPair block : blocks) {
- HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true);
+ HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true, false);
if (buf != null) {
assertEquals(block.block, buf);
}
@@ -149,7 +149,7 @@ public class CacheTestUtils {
for (HFileBlockPair block : blocks) {
try {
- if (toBeTested.getBlock(block.blockName, true) != null) {
+ if (toBeTested.getBlock(block.blockName, true, false) != null) {
toBeTested.cacheBlock(block.blockName, block.block);
fail("Cache should not allow re-caching a block");
}
@@ -179,7 +179,7 @@ public class CacheTestUtils {
@Override
public void doAnAction() throws Exception {
ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested
- .getBlock(key, false);
+ .getBlock(key, false, false);
assertArrayEquals(buf, returned.buf);
totalQueries.incrementAndGet();
}
@@ -218,7 +218,7 @@ public class CacheTestUtils {
final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested
- .getBlock(key, true);
+ .getBlock(key, true, false);
if (gotBack != null) {
assertArrayEquals(gotBack.buf, bac.buf);
} else {
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Tue Nov 13 06:54:16 2012
@@ -240,7 +240,7 @@ public class TestCacheOnWrite {
false, null);
BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
offset, encodingInCache, block.getBlockType());
- boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
+ boolean isCached = blockCache.getBlock(blockCacheKey, true, false) != null;
boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
if (shouldBeCached != isCached) {
throw new AssertionError(
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Tue Nov 13 06:54:16 2012
@@ -98,7 +98,7 @@ public class TestHFileDataBlockEncoder {
BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
blockCache.cacheBlock(cacheKey, cacheBlock);
- HeapSize heapSize = blockCache.getBlock(cacheKey, false);
+ HeapSize heapSize = blockCache.getBlock(cacheKey, false, false);
assertTrue(heapSize instanceof HFileBlock);
HFileBlock returnedBlock = (HFileBlock) heapSize;;
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Tue Nov 13 06:54:16 2012
@@ -116,7 +116,7 @@ public class TestLruBlockCache {
// Confirm empty
for (CachedItem block : blocks) {
- assertTrue(cache.getBlock(block.cacheKey, true) == null);
+ assertTrue(cache.getBlock(block.cacheKey, true, false) == null);
}
// Add blocks
@@ -130,7 +130,7 @@ public class TestLruBlockCache {
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true);
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -150,7 +150,7 @@ public class TestLruBlockCache {
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true);
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -195,10 +195,10 @@ public class TestLruBlockCache {
(maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
// All blocks except block 0 and 1 should be in the cache
- assertTrue(cache.getBlock(blocks[0].cacheKey, true) == null);
- assertTrue(cache.getBlock(blocks[1].cacheKey, true) == null);
+ assertTrue(cache.getBlock(blocks[0].cacheKey, true, false) == null);
+ assertTrue(cache.getBlock(blocks[1].cacheKey, true, false) == null);
for(int i=2;i<blocks.length;i++) {
- assertEquals(cache.getBlock(blocks[i].cacheKey, true),
+ assertEquals(cache.getBlock(blocks[i].cacheKey, true, false),
blocks[i]);
}
}
@@ -220,7 +220,7 @@ public class TestLruBlockCache {
for (CachedItem block : multiBlocks) {
cache.cacheBlock(block.cacheKey, block);
expectedCacheSize += block.cacheBlockHeapSize();
- assertEquals(cache.getBlock(block.cacheKey, true), block);
+ assertEquals(cache.getBlock(block.cacheKey, true, false), block);
}
// Add the single blocks (no get)
@@ -250,14 +250,14 @@ public class TestLruBlockCache {
// This test makes multi go barely over its limit, in-memory
// empty, and the rest in single. Two single evictions and
// one multi eviction expected.
- assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true) == null);
- assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true) == null);
+ assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false) == null);
+ assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false) == null);
// And all others to be cached
for(int i=1;i<4;i++) {
- assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true),
+ assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false),
singleBlocks[i]);
- assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true),
+ assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false),
multiBlocks[i]);
}
}
@@ -295,7 +295,7 @@ public class TestLruBlockCache {
// Add and get multi blocks
cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
- cache.getBlock(multiBlocks[i].cacheKey, true);
+ cache.getBlock(multiBlocks[i].cacheKey, true, false);
// Add memory blocks as such
cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
@@ -317,10 +317,10 @@ public class TestLruBlockCache {
assertEquals(1, cache.getEvictedCount());
// Verify oldest single block is the one evicted
- assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
// Change the oldest remaining single block to a multi
- cache.getBlock(singleBlocks[1].cacheKey, true);
+ cache.getBlock(singleBlocks[1].cacheKey, true, false);
// Insert another single block
cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
@@ -330,7 +330,7 @@ public class TestLruBlockCache {
assertEquals(2, cache.getEvictedCount());
// Oldest multi block should be evicted now
- assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
// Insert another memory block
cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
@@ -340,7 +340,7 @@ public class TestLruBlockCache {
assertEquals(3, cache.getEvictedCount());
// Oldest memory block should be evicted now
- assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false));
// Add a block that is twice as big (should force two evictions)
CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
@@ -351,12 +351,12 @@ public class TestLruBlockCache {
assertEquals(6, cache.getEvictedCount());
// Expect three remaining singles to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true));
- assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true));
- assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false));
// Make the big block a multi block
- cache.getBlock(bigBlocks[0].cacheKey, true);
+ cache.getBlock(bigBlocks[0].cacheKey, true, false);
// Cache another single big block
cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
@@ -366,9 +366,9 @@ public class TestLruBlockCache {
assertEquals(9, cache.getEvictedCount());
// Expect three remaining multis to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
- assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false));
// Cache a big memory block
cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
@@ -378,9 +378,9 @@ public class TestLruBlockCache {
assertEquals(12, cache.getEvictedCount());
// Expect three remaining in-memory to be evicted
- assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true));
- assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true));
- assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false));
}
@@ -408,7 +408,7 @@ public class TestLruBlockCache {
// Add 5 multi blocks
for (CachedItem block : multiBlocks) {
cache.cacheBlock(block.cacheKey, block);
- cache.getBlock(block.cacheKey, true);
+ cache.getBlock(block.cacheKey, true, false);
}
// Add 5 single blocks
@@ -423,10 +423,10 @@ public class TestLruBlockCache {
assertEquals(4, cache.getEvictedCount());
// Should have been taken off equally from single and multi
- assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
- assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
- assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
// Let's keep "scanning" by adding single blocks. From here on we only
// expect evictions from the single bucket.
@@ -477,7 +477,7 @@ public class TestLruBlockCache {
// Add and get multi blocks
cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
- cache.getBlock(multiBlocks[i].cacheKey, true);
+ cache.getBlock(multiBlocks[i].cacheKey, true, false);
// Add memory blocks as such
cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
@@ -497,16 +497,16 @@ public class TestLruBlockCache {
// And the oldest 5 blocks from each category should be gone
for(int i=0;i<5;i++) {
- assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true));
- assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true));
- assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false));
+ assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false));
}
// And the newest 5 blocks should still be accessible
for(int i=5;i<10;i++) {
- assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true));
- assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true));
- assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true));
+ assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false));
+ assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false));
+ assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false));
}
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java?rev=1408621&r1=1408620&r2=1408621&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java Tue Nov 13 06:54:16 2012
@@ -213,7 +213,7 @@ public class TestCacheOnWriteInSchema {
false, null);
BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
offset);
- boolean isCached = cache.getBlock(blockCacheKey, true) != null;
+ boolean isCached = cache.getBlock(blockCacheKey, true, false) != null;
boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
if (shouldBeCached != isCached) {
throw new AssertionError(