svn commit: r1181582 - in /hbase/branches/0.89/src: main/java/org/apache/hadoop/hbase/io/hfile/ test/java/org/apache/hadoop/hbase/io/hfile/

[ns...@apache.org]

Author: nspiegelberg
Date: Tue Oct 11 02:22:34 2011
New Revision: 1181582

URL: http://svn.apache.org/viewvc?rev=1181582&view=rev
Log:
More accurate heapsize measure with new string; fixes unit test fail

Summary: This change includes the size of the string and its unicode characters when determining heapSize() of a HFileBlock

Task ID: #

Blame Rev:#270111

Reviewers: mbautin

CC: kannan

Test Plan: Make sure the unit tests associated with HFileBlock all pass now. Check with mbautin on how much cluster testing is required for this change.

Revert Plan: Revert code, recompile, roll out.

Tags:

Differential Revision: 272589

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java?rev=1181582&r1=1181581&r2=1181582&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java Tue Oct 11 02:22:34 2011
@@ -69,7 +69,13 @@ public class CachedBlock implements Heap
     this.blockName = blockName;
     this.buf = buf;
     this.accessTime = accessTime;
-    this.size = ClassSize.align(blockName.length())
+    // We approximate the size of this class by the size of its name string
+    // plus the size of its byte buffer plus the overhead associated with all
+    // the base classes. Strings have two bytes per character due to default
+    // Java unicode encoding (hence the times 2). We also include the base class
+    // sizes in the PER_BLOCK_OVERHEAD variable rather than align()ing them with
+    // their buffer lengths. This variable is used elsewhere in unit tests.
+    this.size = ClassSize.align(2 * blockName.length())
         + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD;
     if(inMemory) {
       this.priority = BlockPriority.MEMORY;

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1181582&r1=1181581&r2=1181582&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Tue Oct 11 02:22:34 2011
@@ -1380,9 +1380,12 @@ public class HFileBlock implements HeapS
     // uncompressed size, next block's on-disk size, offset and previous
     // offset, byte buffer object, and its byte array. Might also need to add
     // some fields inside the byte buffer.
-    return ClassSize.align(ClassSize.OBJECT + 2 * ClassSize.REFERENCE + 3
-        * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_LONG + BYTE_BUFFER_HEAP_SIZE) +
-        ClassSize.align(buf.capacity());
+    // Also includes size of string cfName, which has its length multiplied by
+    // two because Java characters are unicode and thus 2 bytes each.
+    return ClassSize.align(ClassSize.OBJECT + 3 * ClassSize.REFERENCE + 3
+        * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_LONG)
+        + ClassSize.align(buf.capacity() + BYTE_BUFFER_HEAP_SIZE)
+        + ClassSize.align(ClassSize.STRING + 2 * this.cfName.length());
   }
 
   /**

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1181582&r1=1181581&r2=1181582&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Tue Oct 11 02:22:34 2011
@@ -472,9 +472,11 @@ public class TestHFileBlock {
           true, -1);
       assertEquals(80, HFileBlock.BYTE_BUFFER_HEAP_SIZE);
       long expected = ClassSize.align(ClassSize.estimateBase(HFileBlock.class,
-          true)
-          + ClassSize.estimateBase(buf.getClass(), true)
-          + HFileBlock.HEADER_SIZE + size);
+          true))
+          + ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
+              + HFileBlock.HEADER_SIZE + size)
+          + ClassSize.align(ClassSize.estimateBase(String.class, true) + 2
+              * block.getColumnFamilyName().length());
       assertEquals(expected, block.heapSize());
     }
   }

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1181582&r1=1181581&r2=1181582&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Tue Oct 11 02:22:34 2011
@@ -525,7 +525,7 @@ public class TestLruBlockCache extends T
     /** Size of the cache block holding this item. Used for verification. */
     public long cacheBlockHeapSize() {
       return CachedBlock.PER_BLOCK_OVERHEAD
-          + ClassSize.align(blockName.length())
+          + ClassSize.align(2 * blockName.length())
           + ClassSize.align(size);
     }
   }