You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/07/22 22:17:25 UTC
svn commit: r966840 - in /hbase/trunk: ./ src/assembly/
src/main/java/org/apache/hadoop/hbase/io/hfile/
src/test/java/org/apache/hadoop/hbase/io/hfile/
Author: stack
Date: Thu Jul 22 20:17:24 2010
New Revision: 966840
URL: http://svn.apache.org/viewvc?rev=966840&view=rev
Log:
HBASE-2865 Cleanup of LRU logging; its hard to read, uses custom MB'maker, repeats info, too many numbers after the point, etc.
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/assembly/bin.xml
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Jul 22 20:17:24 2010
@@ -790,6 +790,8 @@ Release 0.21.0 - Unreleased
space and text
HBASE-2850 slf4j version needs to be reconciled in pom: thrift wants 1.5.x
and hadoop/avro 1.4.x
+ HBASE-2865 Cleanup of LRU logging; its hard to read, uses custom MB'maker,
+ repeats info, too many numbers after the point, etc.
NEW FEATURES
HBASE-1961 HBase EC2 scripts
Modified: hbase/trunk/src/assembly/bin.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/assembly/bin.xml?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/assembly/bin.xml (original)
+++ hbase/trunk/src/assembly/bin.xml Thu Jul 22 20:17:24 2010
@@ -28,7 +28,7 @@
<outputDirectory>/</outputDirectory>
<includes>
<include>hbase-${project.version}.jar</include>
- <include>hbase-${project.version}-tests.jar</include>
+ <include>hbase-${project.version}-test.jar</include>
</includes>
</fileSet>
<fileSet>
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java Thu Jul 22 20:17:24 2010
@@ -108,5 +108,4 @@ public class CachedBlock implements Heap
public BlockPriority getPriority() {
return this.priority;
}
-}
-
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java Thu Jul 22 20:17:24 2010
@@ -84,15 +84,14 @@ public class CachedBlockQueue implements
}
/**
- * Get a sorted List of all elements in this queue, in descending order.
- * @return list of cached elements in descending order
+ * @return a sorted List of all elements in this queue, in descending order
*/
- public CachedBlock [] get() {
+ public LinkedList<CachedBlock> get() {
LinkedList<CachedBlock> blocks = new LinkedList<CachedBlock>();
- while(!queue.isEmpty()) {
+ while (!queue.isEmpty()) {
blocks.addFirst(queue.poll());
}
- return blocks.toArray(new CachedBlock[blocks.size()]);
+ return blocks;
}
/**
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Thu Jul 22 20:17:24 2010
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.io.hfile
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
+import java.util.LinkedList;
import java.util.PriorityQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -34,6 +35,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.util.StringUtils;
/**
* A block cache implementation that is memory-aware using {@link HeapSize},
@@ -317,11 +319,14 @@ public class LruBlockCache implements Bl
try {
evictionInProgress = true;
+ long currentSize = this.size.get();
+ long bytesToFree = currentSize - minSize();
- long bytesToFree = size.get() - minSize();
-
- LOG.debug("Block cache LRU eviction started. Attempting to free " +
- bytesToFree + " bytes");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Block cache LRU eviction started; Attempting to free " +
+ StringUtils.byteDesc(bytesToFree) + " of total=" +
+ StringUtils.byteDesc(currentSize));
+ }
if(bytesToFree <= 0) return;
@@ -372,17 +377,17 @@ public class LruBlockCache implements Bl
remainingBuckets--;
}
- float singleMB = ((float)bucketSingle.totalSize())/((float)(1024*1024));
- float multiMB = ((float)bucketMulti.totalSize())/((float)(1024*1024));
- float memoryMB = ((float)bucketMemory.totalSize())/((float)(1024*1024));
-
- LOG.debug("Block cache LRU eviction completed. " +
- "Freed " + bytesFreed + " bytes. " +
- "Priority Sizes: " +
- "Single=" + singleMB + "MB (" + bucketSingle.totalSize() + "), " +
- "Multi=" + multiMB + "MB (" + bucketMulti.totalSize() + ")," +
- "Memory=" + memoryMB + "MB (" + bucketMemory.totalSize() + ")");
-
+ if (LOG.isDebugEnabled()) {
+ long single = bucketSingle.totalSize();
+ long multi = bucketMulti.totalSize();
+ long memory = bucketMemory.totalSize();
+ LOG.debug("Block cache LRU eviction completed; " +
+ "freed=" + StringUtils.byteDesc(bytesFreed) + ", " +
+ "total=" + StringUtils.byteDesc(this.size.get()) + ", " +
+ "single=" + StringUtils.byteDesc(single) + ", " +
+ "multi=" + StringUtils.byteDesc(multi) + ", " +
+ "memory=" + StringUtils.byteDesc(memory));
+ }
} finally {
stats.evict();
evictionInProgress = false;
@@ -414,10 +419,10 @@ public class LruBlockCache implements Bl
}
public long free(long toFree) {
- CachedBlock [] blocks = queue.get();
+ LinkedList<CachedBlock> blocks = queue.get();
long freedBytes = 0;
- for(int i=0; i<blocks.length; i++) {
- freedBytes += evictBlock(blocks[i]);
+ for(CachedBlock cb: blocks) {
+ freedBytes += evictBlock(cb);
if(freedBytes >= toFree) {
return freedBytes;
}
@@ -524,7 +529,7 @@ public class LruBlockCache implements Bl
/*
* Statistics thread. Periodically prints the cache statistics to the log.
*/
- private static class StatisticsThread extends Thread {
+ static class StatisticsThread extends Thread {
LruBlockCache lru;
public StatisticsThread(LruBlockCache lru) {
@@ -539,27 +544,23 @@ public class LruBlockCache implements Bl
}
public void logStats() {
+ if (!LOG.isDebugEnabled()) return;
// Log size
long totalSize = heapSize();
long freeSize = maxSize - totalSize;
- float sizeMB = ((float)totalSize)/((float)(1024*1024));
- float freeMB = ((float)freeSize)/((float)(1024*1024));
- float maxMB = ((float)maxSize)/((float)(1024*1024));
- LruBlockCache.LOG.debug("Cache Stats: Sizes: " +
- "Total=" + sizeMB + "MB (" + totalSize + "), " +
- "Free=" + freeMB + "MB (" + freeSize + "), " +
- "Max=" + maxMB + "MB (" + maxSize +")" +
- ", Counts: " +
- "Blocks=" + size() +", " +
- "Access=" + stats.getRequestCount() + ", " +
- "Hit=" + stats.getHitCount() + ", " +
- "Miss=" + stats.getMissCount() + ", " +
- "Evictions=" + stats.getEvictionCount() + ", " +
- "Evicted=" + stats.getEvictedCount() +
- ", Ratios: " +
- "Hit Ratio=" + stats.getHitRatio()*100 + "%, " +
- "Miss Ratio=" + stats.getMissRatio()*100 + "%, " +
- "Evicted/Run=" + stats.evictedPerEviction());
+ LruBlockCache.LOG.debug("LRU Stats: " +
+ "total=" + StringUtils.byteDesc(totalSize) + ", " +
+ "free=" + StringUtils.byteDesc(freeSize) + ", " +
+ "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
+ "blocks=" + size() +", " +
+ "accesses=" + stats.getRequestCount() + ", " +
+ "hits=" + stats.getHitCount() + ", " +
+ "misses=" + stats.getMissCount() + ", " +
+ "evictions=" + stats.getEvictionCount() + ", " +
+ "evicted=" + stats.getEvictedCount() + ", " +
+ "hitRatio=" + StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, " +
+ "missRatio=" + StringUtils.formatPercent(stats.getMissRatio(), 2) + "%, " +
+ "evictedPerRun=" + stats.evictedPerEviction());
}
/**
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java Thu Jul 22 20:17:24 2010
@@ -20,6 +20,8 @@
package org.apache.hadoop.hbase.io.hfile;
import java.nio.ByteBuffer;
+import java.util.LinkedList;
+
import junit.framework.TestCase;
public class TestCachedBlockQueue extends TestCase {
@@ -57,15 +59,16 @@ public class TestCachedBlockQueue extend
assertEquals(queue.heapSize(), expectedSize);
- org.apache.hadoop.hbase.io.hfile.CachedBlock [] blocks = queue.get();
- assertEquals(blocks[0].getName(), "cb1");
- assertEquals(blocks[1].getName(), "cb2");
- assertEquals(blocks[2].getName(), "cb3");
- assertEquals(blocks[3].getName(), "cb4");
- assertEquals(blocks[4].getName(), "cb5");
- assertEquals(blocks[5].getName(), "cb6");
- assertEquals(blocks[6].getName(), "cb7");
- assertEquals(blocks[7].getName(), "cb8");
+ LinkedList<org.apache.hadoop.hbase.io.hfile.CachedBlock> blocks =
+ queue.get();
+ assertEquals(blocks.poll().getName(), "cb1");
+ assertEquals(blocks.poll().getName(), "cb2");
+ assertEquals(blocks.poll().getName(), "cb3");
+ assertEquals(blocks.poll().getName(), "cb4");
+ assertEquals(blocks.poll().getName(), "cb5");
+ assertEquals(blocks.poll().getName(), "cb6");
+ assertEquals(blocks.poll().getName(), "cb7");
+ assertEquals(blocks.poll().getName(), "cb8");
}
@@ -109,16 +112,16 @@ public class TestCachedBlockQueue extend
assertEquals(queue.heapSize(), expectedSize);
- org.apache.hadoop.hbase.io.hfile.CachedBlock [] blocks = queue.get();
- assertEquals(blocks[0].getName(), "cb0");
- assertEquals(blocks[1].getName(), "cb1");
- assertEquals(blocks[2].getName(), "cb2");
- assertEquals(blocks[3].getName(), "cb3");
- assertEquals(blocks[4].getName(), "cb4");
- assertEquals(blocks[5].getName(), "cb5");
- assertEquals(blocks[6].getName(), "cb6");
- assertEquals(blocks[7].getName(), "cb7");
- assertEquals(blocks[8].getName(), "cb8");
+ LinkedList<org.apache.hadoop.hbase.io.hfile.CachedBlock> blocks = queue.get();
+ assertEquals(blocks.poll().getName(), "cb0");
+ assertEquals(blocks.poll().getName(), "cb1");
+ assertEquals(blocks.poll().getName(), "cb2");
+ assertEquals(blocks.poll().getName(), "cb3");
+ assertEquals(blocks.poll().getName(), "cb4");
+ assertEquals(blocks.poll().getName(), "cb5");
+ assertEquals(blocks.poll().getName(), "cb6");
+ assertEquals(blocks.poll().getName(), "cb7");
+ assertEquals(blocks.poll().getName(), "cb8");
}
@@ -130,5 +133,4 @@ public class TestCachedBlockQueue extend
accessTime,false);
}
}
-
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=966840&r1=966839&r2=966840&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Thu Jul 22 20:17:24 2010
@@ -117,6 +117,9 @@ public class TestLruBlockCache extends T
// Expect no evictions
assertEquals(0, cache.getEvictionCount());
+ Thread t = new LruBlockCache.StatisticsThread(cache);
+ t.start();
+ t.join();
}
public void testCacheEvictionSimple() throws Exception {