You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 19:44:43 UTC
svn commit: r1181967 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/metrics/
test/java/org/apache/hadoop/hbase/io/hfile/
Author: nspiegelberg
Date: Tue Oct 11 17:44:43 2011
New Revision: 1181967
URL: http://svn.apache.org/viewvc?rev=1181967&view=rev
Log:
Porting detailed block cache metrics to hbase-89 from hbase-90
Summary:
Porting the following diffs to hbase-89:
D316730 Add more block cache per-CF and per-BlockType metrics (number evicted,
number hits and misses)
D316912 Reconcile differences in metric naming for block cache / per-CF metrics
before porting to 89
Test Plan:
- Unit tests
- Deploy to a 5-node cluster and see if stats are being collected correctly
Reviewers: jgray, kannan
Reviewed By: kannan
CC: , hbase@lists, kannan, mbautin
Differential Revision: 319078
Revert Plan: OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java Tue Oct 11 17:44:43 2011
@@ -111,8 +111,12 @@ public abstract class AbstractHFileReade
public String fsBlockReadCacheHitCntMetric = "";
public String compactionBlockReadCacheHitCntMetric = "";
+ public String fsBlockReadCacheMissCntMetric = "";
+ public String compactionBlockReadCacheMissCntMetric = "";
+
public String fsMetaBlockReadCntMetric = "";
public String fsMetaBlockReadCacheHitCntMetric = "";
+ public String fsMetaBlockReadCacheMissCntMetric = "";
protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
final FSDataInputStream fsdis, final long fileSize,
@@ -169,13 +173,18 @@ public abstract class AbstractHFileReade
fsBlockReadCntMetric = cfName + ".fsBlockReadCnt";
fsBlockReadCacheHitCntMetric = cfName + ".fsBlockReadCacheHitCnt";
+ fsBlockReadCacheMissCntMetric = cfName + ".fsBlockReadCacheMissCnt";
compactionBlockReadCntMetric = cfName + ".compactionBlockReadCnt";
compactionBlockReadCacheHitCntMetric = cfName
+ ".compactionBlockReadCacheHitCnt";
+ compactionBlockReadCacheMissCntMetric = cfName
+ + ".compactionBlockReadCacheMissCnt";
fsMetaBlockReadCntMetric = cfName + ".fsMetaBlockReadCnt";
fsMetaBlockReadCacheHitCntMetric = cfName + ".fsMetaBlockReadCacheHitCnt";
+ fsMetaBlockReadCacheMissCntMetric = cfName
+ + ".fsMetaBlockReadCacheMissCnt";
}
public abstract boolean isFileInfoLoaded();
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java Tue Oct 11 17:44:43 2011
@@ -45,9 +45,10 @@ public interface BlockCache {
/**
* Fetch block from cache.
* @param blockName Block number to fetch.
+ * @param caching true if the caller caches blocks on a miss
* @return Block or null if block is not in the cache.
*/
- public HeapSize getBlock(String blockName);
+ public HeapSize getBlock(String blockName, boolean caching);
/**
* Evict block from cache.
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockInfo.java Tue Oct 11 17:44:43 2011
@@ -20,8 +20,6 @@
package org.apache.hadoop.hbase.io.hfile;
import org.apache.hadoop.hbase.io.hfile.BlockType;
-import org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer;
-import org.apache.hadoop.hbase.io.HeapSize;
/**
* An interface that exposes methods to retrieve the column type and BlockType
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Tue Oct 11 17:44:43 2011
@@ -223,7 +223,8 @@ public class HFileReaderV1 extends Abstr
HRegion.incrNumericMetric(this.fsMetaBlockReadCntMetric, 1);
// Check cache for block. If found return.
if (blockCache != null) {
- HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey);
+ HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey,
+ cacheBlock);
if (cachedBlock != null) {
cacheHits++;
HRegion.incrNumericMetric(this.fsMetaBlockReadCacheHitCntMetric, 1);
@@ -290,7 +291,8 @@ public class HFileReaderV1 extends Abstr
// Check cache for block. If found return.
if (blockCache != null) {
- HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey);
+ HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey,
+ cacheBlock);
if (cachedBlock != null) {
cacheHits++;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Oct 11 17:44:43 2011
@@ -176,7 +176,8 @@ public class HFileReaderV2 extends Abstr
String cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset);
if (blockCache != null) {
- HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey);
+ HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey,
+ cacheBlock);
if (cachedBlock != null) {
// Return a distinct 'shallow copy' of the block,
// so pos does not get messed by the scanner
@@ -190,6 +191,8 @@ public class HFileReaderV2 extends Abstr
HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
blockSize, -1, true);
metaBlock.setColumnFamilyName(this.getColumnFamilyName());
+ HRegion.incrNumericMetric(fsMetaBlockReadCacheMissCntMetric, 1);
+
long delta = System.currentTimeMillis() - now;
HRegion.incrTimeVaryingMetric(fsReadTimeMetric, delta);
HFile.readTime += delta;
@@ -263,15 +266,21 @@ public class HFileReaderV2 extends Abstr
// Check cache for block. If found return.
if (blockCache != null) {
- HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey);
+ HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey,
+ cacheBlock);
if (cachedBlock != null) {
cacheHits++;
-
if (isCompaction) {
HRegion.incrNumericMetric(
compactionBlockReadCacheHitCntMetric, 1);
+ HRegion.incrNumericMetric("bt." +
+ cachedBlock.getBlockType().getMetricName() +
+ ".compactionBlockReadCacheHitCnt", 1);
} else {
HRegion.incrNumericMetric(fsBlockReadCacheHitCntMetric, 1);
+ HRegion.incrNumericMetric("bt." +
+ cachedBlock.getBlockType().getMetricName() +
+ ".fsBlockReadCacheHitCnt", 1);
}
return cachedBlock;
}
@@ -283,13 +292,18 @@ public class HFileReaderV2 extends Abstr
HFileBlock dataBlock = fsBlockReader.readBlockData(dataBlockOffset,
onDiskBlockSize, -1, pread);
dataBlock.setColumnFamilyName(this.getColumnFamilyName());
+ HRegion.incrNumericMetric("bt." + dataBlock.getBlockType().getMetricName()
+ + ".blockReadCacheMissCnt", 1);
long delta = System.currentTimeMillis() - now;
HFile.readTime += delta;
HFile.readOps++;
if (isCompaction) {
+ HRegion.incrNumericMetric(
+ this.compactionBlockReadCacheMissCntMetric, 1);
HRegion.incrTimeVaryingMetric(compactionReadTimeMetric, delta);
} else {
+ HRegion.incrNumericMetric(this.fsBlockReadCacheMissCntMetric, 1);
HRegion.incrTimeVaryingMetric(fsReadTimeMetric, delta);
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Oct 11 17:44:43 2011
@@ -316,11 +316,29 @@ public class LruBlockCache implements Bl
}
if (cb.getBuffer() instanceof HFileBlockInfo) {
HFileBlockInfo cb_hfbi = (HFileBlockInfo) cb.getBuffer();
+ // CF total size
HRegion.incrNumericPersistentMetric(cb_hfbi.getColumnFamilyName()
+ ".blockCacheSize", heapsize);
+ // BlockType total size
HRegion.incrNumericPersistentMetric("bt."
+ cb_hfbi.getBlockType().getMetricName() + ".blockCacheSize",
heapsize);
+ if (evict) {
+ // CF number evicted
+ HRegion.incrNumericMetric(cb_hfbi.getColumnFamilyName()
+ + ".blockCacheNumEvicted", 1);
+ // BlockType number evicted
+ HRegion.incrNumericMetric("bt." +
+ cb_hfbi.getBlockType().getMetricName() + ".blockCacheNumEvicted",
+ 1);
+ } else {
+ // CF number cached
+ HRegion.incrNumericMetric(cb_hfbi.getColumnFamilyName()
+ + ".blockCacheNumCached", 1);
+ // BlockType number cached
+ HRegion.incrNumericMetric("bt." +
+ cb_hfbi.getBlockType().getMetricName() + ".blockCacheNumCached", 1);
+ }
}
return size.addAndGet(heapsize);
}
@@ -328,16 +346,17 @@ public class LruBlockCache implements Bl
/**
* Get the buffer of the block with the specified name.
* @param blockName block name
+ * @param caching true if the caller caches blocks on cache misses
* @return buffer of specified block name, or null if not in cache
*/
@Override
- public HeapSize getBlock(String blockName) {
+ public HeapSize getBlock(String blockName, boolean caching) {
CachedBlock cb = map.get(blockName);
if(cb == null) {
- stats.miss();
+ stats.miss(caching);
return null;
}
- stats.hit();
+ stats.hit(caching);
cb.access(count.incrementAndGet());
return cb.getBuffer();
}
@@ -373,7 +392,7 @@ public class LruBlockCache implements Bl
map.remove(block.getName());
updateSizeMetrics(block, true);
elements.decrementAndGet();
- stats.evicted();
+ stats.evicted(block);
return block.heapSize();
}
@@ -635,6 +654,8 @@ public class LruBlockCache implements Bl
"Access=" + stats.getRequestCount() + ", " +
"Hit=" + stats.getHitCount() + ", " +
"Miss=" + stats.getMissCount() + ", " +
+ "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
+ "cachingHits=" + stats.getHitCachingCount() + ", " +
"Evictions=" + stats.getEvictionCount() + ", " +
"Evicted=" + stats.getEvictedCount() +
", Ratios: " +
@@ -656,43 +677,84 @@ public class LruBlockCache implements Bl
public static class CacheStats {
private final AtomicLong accessCount = new AtomicLong(0);
private final AtomicLong hitCount = new AtomicLong(0);
+
+ /**
+ * The number of getBlock requests that were cache hits, but only from
+ * requests that were set to use the block cache. This is because all reads
+ * attempt to read from the block cache even if they will not put new blocks
+ * into the block cache. See HBASE-2253 for more information.
+ */
+ private final AtomicLong hitCachingCount = new AtomicLong(0);
+
private final AtomicLong missCount = new AtomicLong(0);
+
+ /**
+ * The number of getBlock requests that were cache misses, but only from
+ * requests that were set to use the block cache.
+ */
+ private final AtomicLong missCachingCount = new AtomicLong(0);
+
private final AtomicLong evictionCount = new AtomicLong(0);
private final AtomicLong evictedCount = new AtomicLong(0);
+ /** The total number of single-access blocks that have been evicted */
+ private final AtomicLong evictedSingleCount = new AtomicLong(0);
+ /** The total number of multi-access blocks that have been evicted */
+ private final AtomicLong evictedMultiCount = new AtomicLong(0);
+ /** The total number of in-memory blocks that have been evicted */
+ private final AtomicLong evictedMemoryCount = new AtomicLong(0);
private long lastHitCount = 0;
private long lastRqCount = 0;
- public void miss() {
+ public void miss(boolean caching) {
missCount.incrementAndGet();
accessCount.incrementAndGet();
+ if (caching) missCachingCount.incrementAndGet();
}
- public void hit() {
+ public void hit(boolean caching) {
hitCount.incrementAndGet();
accessCount.incrementAndGet();
+ if (caching) hitCachingCount.incrementAndGet();
}
public void evict() {
evictionCount.incrementAndGet();
}
- public void evicted() {
+ public void evicted(CachedBlock block) {
evictedCount.incrementAndGet();
+ switch (block.getPriority()) {
+ case SINGLE: evictedSingleCount.incrementAndGet(); break;
+ case MULTI: evictedMultiCount.incrementAndGet(); break;
+ case MEMORY: evictedMemoryCount.incrementAndGet(); break;
+ }
}
public long getRequestCount() {
return accessCount.get();
}
+ public long getRequestCachingCount() {
+ return getHitCachingCount() + getMissCachingCount();
+ }
+
public long getMissCount() {
return missCount.get();
}
+ public long getMissCachingCount() {
+ return missCachingCount.get();
+ }
+
public long getHitCount() {
return hitCount.get();
}
+ public long getHitCachingCount() {
+ return hitCachingCount.get();
+ }
+
public long getEvictionCount() {
return evictionCount.get();
}
@@ -701,6 +763,18 @@ public class LruBlockCache implements Bl
return evictedCount.get();
}
+ public long getEvictedSingleCount() {
+ return evictedSingleCount.get();
+ }
+
+ public long getEvictedMultiCount() {
+ return evictedMultiCount.get();
+ }
+
+ public long getEvictedMemoryCount() {
+ return evictedMemoryCount.get();
+ }
+
public double getHitRatio() {
return ((float)getHitCount()/(float)getRequestCount());
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java Tue Oct 11 17:44:43 2011
@@ -69,7 +69,7 @@ public class SimpleBlockCache implements
return cache.size();
}
- public synchronized HeapSize getBlock(String blockName) {
+ public synchronized HeapSize getBlock(String blockName, boolean caching) {
processQueue(); // clear out some crap.
Ref ref = cache.get(blockName);
if (ref == null)
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Oct 11 17:44:43 2011
@@ -95,6 +95,7 @@ import org.apache.hadoop.hbase.client.Sc
import org.apache.hadoop.hbase.client.ServerConnection;
import org.apache.hadoop.hbase.client.ServerConnectionManager;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
+import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
@@ -1140,8 +1141,18 @@ public class HRegionServer implements HR
this.metrics.blockCacheCount.set(lruBlockCache.size());
this.metrics.blockCacheFree.set(lruBlockCache.getFreeSize());
this.metrics.blockCacheSize.set(lruBlockCache.getCurrentSize());
+ CacheStats cacheStats = lruBlockCache.getStats();
+ this.metrics.blockCacheHitCount.set(cacheStats.getHitCount());
+ this.metrics.blockCacheMissCount.set(cacheStats.getMissCount());
+ this.metrics.blockCacheEvictedCount.set(lruBlockCache.getEvictedCount());
+ this.metrics.blockCacheEvictedSingleCount.set(
+ cacheStats.getEvictedSingleCount());
+ this.metrics.blockCacheEvictedMultiCount.set(
+ cacheStats.getEvictedMultiCount());
+ this.metrics.blockCacheEvictedMemoryCount.set(
+ cacheStats.getEvictedMemoryCount());
- double ratio = lruBlockCache.getStats().getIncrementalHitRatio();
+ double ratio = cacheStats.getIncrementalHitRatio();
int percent = (int) (ratio * 100);
this.metrics.blockCacheHitRatio.set(percent);
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Tue Oct 11 17:44:43 2011
@@ -89,6 +89,24 @@ public class RegionServerMetrics impleme
public final MetricsLongValue blockCacheCount = new MetricsLongValue("blockCacheCount", registry);
/**
+ * Block cache hit count.
+ */
+ public final MetricsLongValue blockCacheHitCount = new MetricsLongValue("blockCacheHitCount", registry);
+
+ /**
+ * Block cache miss count.
+ */
+ public final MetricsLongValue blockCacheMissCount = new MetricsLongValue("blockCacheMissCount", registry);
+
+ /**
+ * Block cache evict count.
+ */
+ public final MetricsLongValue blockCacheEvictedCount = new MetricsLongValue("blockCacheEvictedCount", registry);
+ public final MetricsLongValue blockCacheEvictedSingleCount = new MetricsLongValue("blockCacheEvictedSingleCount", registry);
+ public final MetricsLongValue blockCacheEvictedMultiCount = new MetricsLongValue("blockCacheEvictedMultiCount", registry);
+ public final MetricsLongValue blockCacheEvictedMemoryCount = new MetricsLongValue("blockCacheEvictedMemoryCount", registry);
+
+ /**
* Block hit ratio.
*/
public final MetricsIntValue blockCacheHitRatio = new MetricsIntValue("blockCacheHitRatio", registry);
@@ -265,6 +283,12 @@ public class RegionServerMetrics impleme
this.blockCacheSize.pushMetric(this.metricsRecord);
this.blockCacheFree.pushMetric(this.metricsRecord);
this.blockCacheCount.pushMetric(this.metricsRecord);
+ this.blockCacheHitCount.pushMetric(this.metricsRecord);
+ this.blockCacheMissCount.pushMetric(this.metricsRecord);
+ this.blockCacheEvictedCount.pushMetric(this.metricsRecord);
+ this.blockCacheEvictedSingleCount.pushMetric(this.metricsRecord);
+ this.blockCacheEvictedMultiCount.pushMetric(this.metricsRecord);
+ this.blockCacheEvictedMemoryCount.pushMetric(this.metricsRecord);
this.blockCacheHitRatio.pushMetric(this.metricsRecord);
// Be careful. Here is code for MTVR from up in hadoop:
@@ -412,6 +436,18 @@ public class RegionServerMetrics impleme
Long.valueOf(this.blockCacheFree.get()));
sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(),
Long.valueOf(this.blockCacheCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(),
+ Long.valueOf(this.blockCacheHitCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(),
+ Long.valueOf(this.blockCacheMissCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(),
+ Long.valueOf(this.blockCacheEvictedCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheEvictedSingleCount.getName(),
+ Long.valueOf(this.blockCacheEvictedSingleCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheEvictedMultiCount.getName(),
+ Long.valueOf(this.blockCacheEvictedMultiCount.get()));
+ sb = Strings.appendKeyValue(sb, this.blockCacheEvictedMemoryCount.getName(),
+ Long.valueOf(this.blockCacheEvictedMemoryCount.get()));
sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
Long.valueOf(this.blockCacheHitRatio.get()));
return sb.toString();
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Tue Oct 11 17:44:43 2011
@@ -163,7 +163,7 @@ public class TestCacheOnWrite {
HFileBlock block = reader.readBlockData(offset, prevBlock == null ? -1
: prevBlock.getNextBlockOnDiskSizeWithHeader(), -1, false);
String blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset);
- boolean isCached = blockCache.getBlock(blockCacheKey) != null;
+ boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
assertEquals(testName + " " + block, shouldBeCached, isCached);
prevBlock = block;
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1181967&r1=1181966&r2=1181967&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Tue Oct 11 17:44:43 2011
@@ -75,7 +75,7 @@ public class TestLruBlockCache extends T
// Confirm empty
for (CachedItem block : blocks) {
- assertTrue(cache.getBlock(block.blockName) == null);
+ assertTrue(cache.getBlock(block.blockName, true) == null);
}
// Add blocks
@@ -89,7 +89,7 @@ public class TestLruBlockCache extends T
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.blockName);
+ HeapSize buf = cache.getBlock(block.blockName, true);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -109,7 +109,7 @@ public class TestLruBlockCache extends T
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.blockName);
+ HeapSize buf = cache.getBlock(block.blockName, true);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -150,10 +150,10 @@ public class TestLruBlockCache extends T
(maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
// All blocks except block 0 and 1 should be in the cache
- assertTrue(cache.getBlock(blocks[0].blockName) == null);
- assertTrue(cache.getBlock(blocks[1].blockName) == null);
+ assertTrue(cache.getBlock(blocks[0].blockName, true) == null);
+ assertTrue(cache.getBlock(blocks[1].blockName, true) == null);
for(int i=2;i<blocks.length;i++) {
- assertEquals(cache.getBlock(blocks[i].blockName),
+ assertEquals(cache.getBlock(blocks[i].blockName, true),
blocks[i]);
}
}
@@ -174,7 +174,7 @@ public class TestLruBlockCache extends T
for (CachedItem block : multiBlocks) {
cache.cacheBlock(block.blockName, block);
expectedCacheSize += block.cacheBlockHeapSize();
- assertEquals(cache.getBlock(block.blockName), block);
+ assertEquals(cache.getBlock(block.blockName, true), block);
}
// Add the single blocks (no get)
@@ -204,14 +204,14 @@ public class TestLruBlockCache extends T
// This test makes multi go barely over its limit, in-memory
// empty, and the rest in single. Two single evictions and
// one multi eviction expected.
- assertTrue(cache.getBlock(singleBlocks[0].blockName) == null);
- assertTrue(cache.getBlock(multiBlocks[0].blockName) == null);
+ assertTrue(cache.getBlock(singleBlocks[0].blockName, true) == null);
+ assertTrue(cache.getBlock(multiBlocks[0].blockName, true) == null);
// And all others to be cached
for(int i=1;i<4;i++) {
- assertEquals(cache.getBlock(singleBlocks[i].blockName),
+ assertEquals(cache.getBlock(singleBlocks[i].blockName, true),
singleBlocks[i]);
- assertEquals(cache.getBlock(multiBlocks[i].blockName),
+ assertEquals(cache.getBlock(multiBlocks[i].blockName, true),
multiBlocks[i]);
}
}
@@ -248,7 +248,7 @@ public class TestLruBlockCache extends T
// Add and get multi blocks
cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
- cache.getBlock(multiBlocks[i].blockName);
+ cache.getBlock(multiBlocks[i].blockName, true);
// Add memory blocks as such
cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
@@ -270,10 +270,10 @@ public class TestLruBlockCache extends T
assertEquals(1, cache.getEvictedCount());
// Verify oldest single block is the one evicted
- assertEquals(null, cache.getBlock(singleBlocks[0].blockName));
+ assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
// Change the oldest remaining single block to a multi
- cache.getBlock(singleBlocks[1].blockName);
+ cache.getBlock(singleBlocks[1].blockName, true);
// Insert another single block
cache.cacheBlock(singleBlocks[4].blockName, singleBlocks[4]);
@@ -283,7 +283,7 @@ public class TestLruBlockCache extends T
assertEquals(2, cache.getEvictedCount());
// Oldest multi block should be evicted now
- assertEquals(null, cache.getBlock(multiBlocks[0].blockName));
+ assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
// Insert another memory block
cache.cacheBlock(memoryBlocks[3].blockName, memoryBlocks[3], true);
@@ -293,7 +293,7 @@ public class TestLruBlockCache extends T
assertEquals(3, cache.getEvictedCount());
// Oldest memory block should be evicted now
- assertEquals(null, cache.getBlock(memoryBlocks[0].blockName));
+ assertEquals(null, cache.getBlock(memoryBlocks[0].blockName, true));
// Add a block that is twice as big (should force two evictions)
CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
@@ -304,12 +304,12 @@ public class TestLruBlockCache extends T
assertEquals(6, cache.getEvictedCount());
// Expect three remaining singles to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[2].blockName));
- assertEquals(null, cache.getBlock(singleBlocks[3].blockName));
- assertEquals(null, cache.getBlock(singleBlocks[4].blockName));
+ assertEquals(null, cache.getBlock(singleBlocks[2].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[3].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[4].blockName, true));
// Make the big block a multi block
- cache.getBlock(bigBlocks[0].blockName);
+ cache.getBlock(bigBlocks[0].blockName, true);
// Cache another single big block
cache.cacheBlock(bigBlocks[1].blockName, bigBlocks[1]);
@@ -319,9 +319,9 @@ public class TestLruBlockCache extends T
assertEquals(9, cache.getEvictedCount());
// Expect three remaining multis to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[1].blockName));
- assertEquals(null, cache.getBlock(multiBlocks[1].blockName));
- assertEquals(null, cache.getBlock(multiBlocks[2].blockName));
+ assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[2].blockName, true));
// Cache a big memory block
cache.cacheBlock(bigBlocks[2].blockName, bigBlocks[2], true);
@@ -331,9 +331,9 @@ public class TestLruBlockCache extends T
assertEquals(12, cache.getEvictedCount());
// Expect three remaining in-memory to be evicted
- assertEquals(null, cache.getBlock(memoryBlocks[1].blockName));
- assertEquals(null, cache.getBlock(memoryBlocks[2].blockName));
- assertEquals(null, cache.getBlock(memoryBlocks[3].blockName));
+ assertEquals(null, cache.getBlock(memoryBlocks[1].blockName, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[2].blockName, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[3].blockName, true));
}
@@ -360,7 +360,7 @@ public class TestLruBlockCache extends T
// Add 5 multi blocks
for (CachedItem block : multiBlocks) {
cache.cacheBlock(block.blockName, block);
- cache.getBlock(block.blockName);
+ cache.getBlock(block.blockName, true);
}
// Add 5 single blocks
@@ -375,10 +375,10 @@ public class TestLruBlockCache extends T
assertEquals(4, cache.getEvictedCount());
// Should have been taken off equally from single and multi
- assertEquals(null, cache.getBlock(singleBlocks[0].blockName));
- assertEquals(null, cache.getBlock(singleBlocks[1].blockName));
- assertEquals(null, cache.getBlock(multiBlocks[0].blockName));
- assertEquals(null, cache.getBlock(multiBlocks[1].blockName));
+ assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
// Let's keep "scanning" by adding single blocks. From here on we only
// expect evictions from the single bucket.
@@ -428,7 +428,7 @@ public class TestLruBlockCache extends T
// Add and get multi blocks
cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
- cache.getBlock(multiBlocks[i].blockName);
+ cache.getBlock(multiBlocks[i].blockName, true);
// Add memory blocks as such
cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
@@ -448,16 +448,19 @@ public class TestLruBlockCache extends T
// And the oldest 5 blocks from each category should be gone
for(int i=0;i<5;i++) {
- assertEquals(null, cache.getBlock(singleBlocks[i].blockName));
- assertEquals(null, cache.getBlock(multiBlocks[i].blockName));
- assertEquals(null, cache.getBlock(memoryBlocks[i].blockName));
+ assertEquals(null, cache.getBlock(singleBlocks[i].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[i].blockName, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[i].blockName, true));
}
// And the newest 5 blocks should still be accessible
for(int i=5;i<10;i++) {
- assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].blockName));
- assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].blockName));
- assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].blockName));
+ assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].blockName,
+ true));
+ assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].blockName,
+ true));
+ assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].blockName,
+ true));
}
}