You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/02/07 23:51:14 UTC
svn commit: r1241679 - in /hbase/branches/0.89-fb: ./
src/main/java/org/apache/hadoop/hbase/io/hfile/
src/main/java/org/apache/hadoop/hbase/regionserver/wal/
src/main/java/org/apache/hadoop/hbase/util/
src/test/java/org/apache/hadoop/hbase/io/hfile/ sr...
Author: mbautin
Date: Tue Feb 7 22:51:13 2012
New Revision: 1241679
URL: http://svn.apache.org/viewvc?rev=1241679&view=rev
Log:
[jira] [HBASE-5001] [89-fb] Improve the performance of block cache keys (J-D, Lars H; pulling from trunk)
Summary: Pulling the BlockCacheKey patch
(https://issues.apache.org/jira/browse/HBASE-5001) and associated unit tests
and dependencies from trunk. Had to upgrade Guava dependency to r09. Kept
original formatting in most places.
Test Plan: Unit tests, dev cluster
Reviewers: kannan, liyintang, nspiegelberg, jdcryans, stack, lhofhansl, khemani
Reviewed By: khemani
CC: mbautin, khemani
Differential Revision: https://reviews.facebook.net/D1587
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java
Modified:
hbase/branches/0.89-fb/pom.xml
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
Modified: hbase/branches/0.89-fb/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/pom.xml?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/pom.xml (original)
+++ hbase/branches/0.89-fb/pom.xml Tue Feb 7 22:51:13 2012
@@ -490,7 +490,7 @@
<slf4j.version>1.5.8</slf4j.version>
<stax-api>1.0.1</stax-api>
<thrift.version>0.2.0</thrift.version>
- <guava.version>r05</guava.version>
+ <guava.version>r09</guava.version>
</properties>
<dependencyManagement>
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java Tue Feb 7 22:51:13 2012
@@ -19,48 +19,53 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
/**
- * Block cache interface.
- * TODO: Add filename or hash of filename to block cache key.
+ * Block cache interface. Anything that implements the {@link Cacheable}
+ * interface can be put in the cache.
*/
public interface BlockCache {
/**
* Add block to cache.
- * @param blockName Zero-based file block number.
+ * @param cacheKey The block's cache key.
* @param buf The block contents wrapped in a ByteBuffer.
* @param inMemory Whether block should be treated as in-memory
*/
- public void cacheBlock(String blockName, Cacheable buf, boolean inMemory);
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory);
/**
* Add block to cache (defaults to not in-memory).
- * @param blockName Zero-based file block number.
- * @param buf The block contents wrapped in a ByteBuffer.
+ * @param cacheKey The block's cache key.
+ * @param buf The object to cache.
*/
- public void cacheBlock(String blockName, Cacheable buf);
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf);
/**
* Fetch block from cache.
- * @param blockName Block number to fetch.
- * @param caching true if the caller caches blocks on a miss
+ * @param cacheKey Block to fetch.
+ * @param caching Whether this request has caching enabled (used for stats)
* @return Block or null if block is not in the cache.
*/
- public Cacheable getBlock(String blockName, boolean caching);
+ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching);
/**
* Evict block from cache.
- * @param blockName Block name to evict
+ * @param cacheKey Block to evict
* @return true if block existed and was evicted, false if not
*/
- public boolean evictBlock(String blockName);
+ public boolean evictBlock(BlockCacheKey cacheKey);
/**
- * Evicts all blocks with the given prefix in the name
+ * Evicts all blocks for the given HFile.
+ *
* @return the number of blocks evicted
*/
- public int evictBlocksByPrefix(String string);
+ public int evictBlocksByHfileName(String hfileName);
/**
* Get the statistics for this block cache.
@@ -74,8 +79,47 @@ public interface BlockCache {
public void shutdown();
/**
+ * Returns the total size of the block cache, in bytes.
+ * @return size of cache, in bytes
+ */
+ public long size();
+
+ /**
+ * Returns the free size of the block cache, in bytes.
+ * @return free space in cache, in bytes
+ */
+ public long getFreeSize();
+
+ /**
+ * Returns the occupied size of the block cache, in bytes.
+ * @return occupied space in cache, in bytes
+ */
+ public long getCurrentSize();
+
+ /**
+ * Returns the number of evictions that have occurred.
+ * @return number of evictions
+ */
+ public long getEvictedCount();
+
+ /**
* Returns the number of blocks currently cached in the block cache.
* @return number of blocks in the cache
*/
public long getBlockCount();
+
+ /**
+ * Performs a BlockCache summary and returns a List of BlockCacheColumnFamilySummary objects.
+ * This method could be fairly heavyweight in that it evaluates the entire HBase file-system
+ * against what is in the RegionServer BlockCache.
+ * <br><br>
+ * The contract of this interface is to return the List in sorted order by Table name, then
+ * ColumnFamily.
+ *
+ * @param conf HBaseConfiguration
+ * @return List of BlockCacheColumnFamilySummary
+ * @throws IOException exception
+ */
+ public List<BlockCacheColumnFamilySummary> getBlockCacheColumnFamilySummaries(
+ Configuration conf) throws IOException;
}
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java?rev=1241679&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java Tue Feb 7 22:51:13 2012
@@ -0,0 +1,246 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * BlockCacheColumnFamilySummary represents a summary of the blockCache usage
+ * at Table/ColumnFamily granularity.
+ * <br><br>
+ * As ColumnFamilies are owned by Tables, a summary by ColumnFamily implies that
+ * the owning Table is included in the summarization.
+ *
+ */
+public class BlockCacheColumnFamilySummary implements Writable, Comparable<BlockCacheColumnFamilySummary> {
+
+ private String table = "";
+ private String columnFamily = "";
+ private int blocks;
+ private long heapSize;
+
+ /**
+ * Default constructor for Writable
+ */
+ public BlockCacheColumnFamilySummary() {
+
+ }
+
+ /**
+ *
+ * @param table table
+ * @param columnFamily columnFamily
+ */
+ public BlockCacheColumnFamilySummary(String table, String columnFamily) {
+ this.table = table;
+ this.columnFamily = columnFamily;
+ }
+
+ /**
+ *
+ * @return table
+ */
+ public String getTable() {
+ return table;
+ }
+ /**
+ *
+ * @param table (table that owns the cached block)
+ */
+ public void setTable(String table) {
+ this.table = table;
+ }
+ /**
+ *
+ * @return columnFamily
+ */
+ public String getColumnFamily() {
+ return columnFamily;
+ }
+ /**
+ *
+ * @param columnFamily (columnFamily that owns the cached block)
+ */
+ public void setColumnFamily(String columnFamily) {
+ this.columnFamily = columnFamily;
+ }
+
+ /**
+ *
+ * @return blocks in the cache
+ */
+ public int getBlocks() {
+ return blocks;
+ }
+ /**
+ *
+ * @param blocks in the cache
+ */
+ public void setBlocks(int blocks) {
+ this.blocks = blocks;
+ }
+
+ /**
+ *
+ * @return heapSize in the cache
+ */
+ public long getHeapSize() {
+ return heapSize;
+ }
+
+ /**
+ * Increments the number of blocks in the cache for this entry
+ */
+ public void incrementBlocks() {
+ this.blocks++;
+ }
+
+ /**
+ *
+ * @param heapSize to increment
+ */
+ public void incrementHeapSize(long heapSize) {
+ this.heapSize = this.heapSize + heapSize;
+ }
+
+ /**
+ *
+ * @param heapSize (total heapSize for the table/CF)
+ */
+ public void setHeapSize(long heapSize) {
+ this.heapSize = heapSize;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ table = in.readUTF();
+ columnFamily = in.readUTF();
+ blocks = in.readInt();
+ heapSize = in.readLong();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(table);
+ out.writeUTF(columnFamily);
+ out.writeInt(blocks);
+ out.writeLong(heapSize);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((columnFamily == null) ? 0 : columnFamily.hashCode());
+ result = prime * result + ((table == null) ? 0 : table.hashCode());
+ return result;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ BlockCacheColumnFamilySummary other = (BlockCacheColumnFamilySummary) obj;
+ if (columnFamily == null) {
+ if (other.columnFamily != null)
+ return false;
+ } else if (!columnFamily.equals(other.columnFamily))
+ return false;
+ if (table == null) {
+ if (other.table != null)
+ return false;
+ } else if (!table.equals(other.table))
+ return false;
+ return true;
+ }
+
+
+
+ @Override
+ public String toString() {
+ return "BlockCacheSummaryEntry [table=" + table + ", columnFamily="
+ + columnFamily + ", blocks=" + blocks + ", heapSize=" + heapSize + "]";
+ }
+
+ /**
+ * Construct a BlockCacheSummaryEntry from a full StoreFile Path
+ * <br><br>
+ * The path is expected to be in the format of...
+ * <pre>
+ * hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ * </pre>
+ * ... where: <br>
+ * '-ROOT-' = Table <br>
+ * '70236052' = Region <br>
+ * 'info' = ColumnFamily <br>
+ * '3944417774205889744' = StoreFile
+ *
+ * @param path (full StoreFile Path)
+ * @return BlockCacheSummaryEntry
+ */
+ public static BlockCacheColumnFamilySummary createFromStoreFilePath(Path path) {
+
+ // The full path will look something like this...
+ // hdfs://localhost:51169/user/doug.meil/-ROOT-/70236052/info/3944417774205889744
+ // tbl region cf sf
+ String sp = path.toString();
+ String s[] = sp.split("\\/");
+
+ BlockCacheColumnFamilySummary bcse = null;
+ if (s.length >= 4) {
+ // why 4? StoreFile, CF, Region, Table
+ String table = s[s.length - 4]; // 4th from the end
+ String cf = s[s.length - 2]; // 2nd from the end
+ bcse = new BlockCacheColumnFamilySummary(table, cf);
+ }
+ return bcse;
+ }
+
+ @Override
+ public int compareTo(BlockCacheColumnFamilySummary o) {
+ int i = table.compareTo(o.getTable());
+ if (i != 0) {
+ return i;
+ }
+ return columnFamily.compareTo(o.getColumnFamily());
+ }
+
+ /**
+ * Creates a new BlockCacheSummaryEntry
+ *
+ * @param e BlockCacheSummaryEntry
+ * @return new BlockCacheSummaryEntry
+ */
+ public static BlockCacheColumnFamilySummary create(BlockCacheColumnFamilySummary e) {
+ BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary();
+ e2.setTable(e.getTable());
+ e2.setColumnFamily(e.getColumnFamily());
+ return e2;
+ }
+}
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java?rev=1241679&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java Tue Feb 7 22:51:13 2012
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Cache Key for use with implementations of {@link BlockCache}
+ */
+public class BlockCacheKey implements HeapSize {
+ private String hfileName;
+ private long offset;
+
+ /**
+ * Construct a new BlockCacheKey
+ * @param file The name of the HFile this block belongs to.
+ * @param offset Offset of the block into the file
+ */
+ public BlockCacheKey(String file, long offset) {
+ this.hfileName = file;
+ this.offset = offset;
+ }
+
+ @Override
+ public int hashCode() {
+ return hfileName.hashCode() * 127 + (int) (offset ^ (offset >>> 32));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof BlockCacheKey) {
+ BlockCacheKey k = (BlockCacheKey) o;
+ return offset == k.offset
+ && (hfileName == null ? k.hfileName == null : hfileName
+ .equals(k.hfileName));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return hfileName + "_" + offset;
+ }
+
+ // Strings have two bytes per character due to default
+ // Java unicode encoding (hence the times 2).
+ @Override
+ public long heapSize() {
+ return 2 * hfileName.length() + Bytes.SIZEOF_LONG;
+ }
+
+ // can't avoid this unfortunately
+ /**
+ * @return The hfileName portion of this cache key
+ */
+ public String getHfileName() {
+ return hfileName;
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java Tue Feb 7 22:51:13 2012
@@ -52,28 +52,27 @@ public class CachedBlock implements Heap
MEMORY
};
- private final String blockName;
+ private final BlockCacheKey cacheKey;
private final Cacheable buf;
private volatile long accessTime;
private long size;
private BlockPriority priority;
- public CachedBlock(String blockName, Cacheable buf, long accessTime) {
- this(blockName, buf, accessTime, false);
+ public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime) {
+ this(cacheKey, buf, accessTime, false);
}
- public CachedBlock(String blockName, Cacheable buf, long accessTime,
+ public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime,
boolean inMemory) {
- this.blockName = blockName;
+ this.cacheKey = cacheKey;
this.buf = buf;
this.accessTime = accessTime;
// We approximate the size of this class by the size of its name string
// plus the size of its byte buffer plus the overhead associated with all
- // the base classes. Strings have two bytes per character due to default
- // Java unicode encoding (hence the times 2). We also include the base class
+ // the base classes. We also include the base class
// sizes in the PER_BLOCK_OVERHEAD variable rather than align()ing them with
// their buffer lengths. This variable is used elsewhere in unit tests.
- this.size = ClassSize.align(2 * blockName.length())
+ this.size = ClassSize.align(cacheKey.heapSize())
+ ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD;
if(inMemory) {
this.priority = BlockPriority.MEMORY;
@@ -105,12 +104,11 @@ public class CachedBlock implements Heap
return this.buf;
}
- public String getName() {
- return this.blockName;
+ public BlockCacheKey getCacheKey() {
+ return this.cacheKey;
}
public BlockPriority getPriority() {
return this.priority;
}
}
-
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java Tue Feb 7 22:51:13 2012
@@ -20,7 +20,8 @@
package org.apache.hadoop.hbase.io.hfile;
import java.util.LinkedList;
-import java.util.PriorityQueue;
+
+import com.google.common.collect.MinMaxPriorityQueue;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -39,7 +40,7 @@ import org.apache.hadoop.hbase.io.HeapSi
*/
public class CachedBlockQueue implements HeapSize {
- private PriorityQueue<CachedBlock> queue;
+ private MinMaxPriorityQueue<CachedBlock> queue;
private long heapSize;
private long maxSize;
@@ -51,7 +52,7 @@ public class CachedBlockQueue implements
public CachedBlockQueue(long maxSize, long blockSize) {
int initialSize = (int)(maxSize / blockSize);
if(initialSize == 0) initialSize++;
- queue = new PriorityQueue<CachedBlock>(initialSize);
+ queue = MinMaxPriorityQueue.expectedSize(initialSize).create();
heapSize = 0;
this.maxSize = maxSize;
}
@@ -96,6 +97,14 @@ public class CachedBlockQueue implements
}
/**
+ * @return The last element in this queue, or {@code null} if the queue is
+ * empty.
+ */
+ public CachedBlock pollLast() {
+ return queue.pollLast();
+ }
+
+ /**
* Total size of all elements in this queue.
* @return size of all elements currently in queue, in bytes
*/
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Feb 7 22:51:13 2012
@@ -144,9 +144,6 @@ public class HFile {
public final static String DEFAULT_COMPRESSION =
DEFAULT_COMPRESSION_ALGORITHM.getName();
- /** Separator between HFile name and offset in block cache key */
- static final char CACHE_KEY_SEPARATOR = '_';
-
/**
* ROOT_DIR/TABLE_NAME/REGION_NAME/CF_NAME/HFILE
*/
@@ -516,8 +513,8 @@ public class HFile {
System.exit(prettyPrinter.run(args));
}
- public static String getBlockCacheKey(String hfileName, long offset) {
- return hfileName + CACHE_KEY_SEPARATOR + offset;
+ public static BlockCacheKey getBlockCacheKey(String hfileName, long offset) {
+ return new BlockCacheKey(hfileName, offset);
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Tue Feb 7 22:51:13 2012
@@ -215,7 +215,7 @@ public class HFileReaderV1 extends Abstr
long now = System.currentTimeMillis();
- String cacheKey = HFile.getBlockCacheKey(name, offset);
+ BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, offset);
BlockCategory effectiveCategory = BlockCategory.META;
if (metaBlockName.equals(HFileWriterV1.BLOOM_FILTER_META_KEY) ||
@@ -279,7 +279,7 @@ public class HFileReaderV1 extends Abstr
}
long offset = dataBlockIndexReader.getRootBlockOffset(block);
- String cacheKey = HFile.getBlockCacheKey(name, offset);
+ BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, offset);
// For any given block from any given file, synchronize reads for said
// block.
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Feb 7 22:51:13 2012
@@ -179,7 +179,7 @@ public class HFileReaderV2 extends Abstr
synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
// Check cache for block. If found return.
long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
- String cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset);
+ BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset);
cacheBlock &= cacheConf.shouldCacheDataOnRead();
if (cacheConf.isBlockCacheEnabled()) {
@@ -242,7 +242,7 @@ public class HFileReaderV2 extends Abstr
// the other choice is to duplicate work (which the cache would prevent you
// from doing).
- String cacheKey = HFile.getBlockCacheKey(name, dataBlockOffset);
+ BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, dataBlockOffset);
IdLock.Entry lockEntry = offsetLock.getLockEntry(dataBlockOffset);
try {
// Check cache for block. If found return.
@@ -310,11 +310,12 @@ public class HFileReaderV2 extends Abstr
@Override
public void close(boolean evictOnClose) throws IOException {
- if (evictOnClose && cacheConf.shouldEvictOnClose()) {
- int numEvicted = cacheConf.getBlockCache().evictBlocksByPrefix(name
- + HFile.CACHE_KEY_SEPARATOR);
- LOG.debug("On close of file " + name + " evicted " + numEvicted
+ if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
+ int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("On close, file=" + name + " evicted=" + numEvicted
+ " block(s)");
+ }
}
if (closeIStream && istream != null) {
istream.close();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Feb 7 22:51:13 2012
@@ -19,8 +19,13 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import java.io.IOException;
import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.SortedSet;
@@ -35,10 +40,13 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.FSUtils;
/**
* A block cache implementation that is memory-aware using {@link HeapSize},
@@ -101,7 +109,7 @@ public class LruBlockCache implements Bl
static final int statThreadPeriod = 60;
/** Concurrent map (the cache) */
- private final ConcurrentHashMap<String,CachedBlock> map;
+ private final ConcurrentHashMap<BlockCacheKey,CachedBlock> map;
/** Eviction lock (locked when eviction in process) */
private final ReentrantLock evictionLock = new ReentrantLock(true);
@@ -234,7 +242,7 @@ public class LruBlockCache implements Bl
}
this.maxSize = maxSize;
this.blockSize = blockSize;
- map = new ConcurrentHashMap<String,CachedBlock>(mapInitialSize,
+ map = new ConcurrentHashMap<BlockCacheKey,CachedBlock>(mapInitialSize,
mapLoadFactor, mapConcurrencyLevel);
this.minFactor = minFactor;
this.acceptableFactor = acceptableFactor;
@@ -270,18 +278,18 @@ public class LruBlockCache implements Bl
* <p>
* It is assumed this will NEVER be called on an already cached block. If
* that is done, an exception will be thrown.
- * @param blockName block name
+ * @param cacheKey block's cache key
* @param buf block buffer
* @param inMemory if block is in-memory
*/
- public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) {
- CachedBlock cb = map.get(blockName);
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+ CachedBlock cb = map.get(cacheKey);
if(cb != null) {
throw new RuntimeException("Cached an already cached block");
}
- cb = new CachedBlock(blockName, buf, count.incrementAndGet(), inMemory);
+ cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
long newSize = updateSizeMetrics(cb, false);
- map.put(blockName, cb);
+ map.put(cacheKey, cb);
elements.incrementAndGet();
if(newSize > acceptableSize() && !evictionInProgress) {
runEviction();
@@ -295,11 +303,11 @@ public class LruBlockCache implements Bl
* that is done, it is assumed that you are reinserting the same exact
* block due to a race condition and will update the buffer but not modify
* the size of the cache.
- * @param blockName block name
+ * @param cacheKey block's cache key
* @param buf block buffer
*/
- public void cacheBlock(String blockName, Cacheable buf) {
- cacheBlock(blockName, buf, false);
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
+ cacheBlock(cacheKey, buf, false);
}
/**
@@ -326,13 +334,13 @@ public class LruBlockCache implements Bl
/**
* Get the buffer of the block with the specified name.
- * @param blockName block name
+ * @param cacheKey block's cache key
* @param caching true if the caller caches blocks on cache misses
- * @return buffer of specified block name, or null if not in cache
+ * @return buffer of specified cache key, or null if not in cache
*/
@Override
- public Cacheable getBlock(String blockName, boolean caching) {
- CachedBlock cb = map.get(blockName);
+ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+ CachedBlock cb = map.get(cacheKey);
if(cb == null) {
stats.miss(caching);
return null;
@@ -343,25 +351,28 @@ public class LruBlockCache implements Bl
}
@Override
- public boolean evictBlock(String blockName) {
- CachedBlock cb = map.get(blockName);
+ public boolean evictBlock(BlockCacheKey cacheKey) {
+ CachedBlock cb = map.get(cacheKey);
if (cb == null) return false;
evictBlock(cb);
return true;
}
/**
- * Evicts all blocks whose name starts with the given prefix. This is an
+ * Evicts all blocks for a specific HFile. This is an
* expensive operation implemented as a linear-time search through all blocks
- * in the cache.
+ * in the cache. Ideally this should be a search in a log-access-time map.
+ *
+ * <p>
+ * This is used for evict-on-close to remove all blocks of a specific HFile.
*
* @return the number of blocks evicted
*/
@Override
- public int evictBlocksByPrefix(String prefix) {
+ public int evictBlocksByHfileName(String hfileName) {
int numEvicted = 0;
- for (String key : map.keySet()) {
- if (key.startsWith(prefix)) {
+ for (BlockCacheKey key : map.keySet()) {
+ if (key.getHfileName().equals(hfileName)) {
if (evictBlock(key))
++numEvicted;
}
@@ -370,7 +381,7 @@ public class LruBlockCache implements Bl
}
protected long evictBlock(CachedBlock block) {
- map.remove(block.getName());
+ map.remove(block.getCacheKey());
updateSizeMetrics(block, true);
elements.decrementAndGet();
stats.evicted(block);
@@ -805,6 +816,40 @@ public class LruBlockCache implements Bl
(concurrency * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
}
+ @Override
+ public List<BlockCacheColumnFamilySummary> getBlockCacheColumnFamilySummaries(
+ Configuration conf) throws IOException {
+
+ Map<String, Path> sfMap = FSUtils.getTableStoreFilePathMap(
+ FileSystem.get(conf),
+ FSUtils.getRootDir(conf));
+
+ // quirky, but it's a compound key and this is a shortcut taken instead of
+ // creating a class that would represent only a key.
+ Map<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary> bcs =
+ new HashMap<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary>();
+
+ for (CachedBlock cb : map.values()) {
+ String sf = cb.getCacheKey().getHfileName();
+ Path path = sfMap.get(sf);
+ if ( path != null) {
+ BlockCacheColumnFamilySummary lookup =
+ BlockCacheColumnFamilySummary.createFromStoreFilePath(path);
+ BlockCacheColumnFamilySummary bcse = bcs.get(lookup);
+ if (bcse == null) {
+ bcse = BlockCacheColumnFamilySummary.create(lookup);
+ bcs.put(lookup,bcse);
+ }
+ bcse.incrementBlocks();
+ bcse.incrementHeapSize(cb.heapSize());
+ }
+ }
+ List<BlockCacheColumnFamilySummary> list =
+ new ArrayList<BlockCacheColumnFamilySummary>(bcs.values());
+ Collections.sort( list );
+ return list;
+ }
+
// Simple calculators of sizes given factors and maxSize
private long acceptableSize() {
@@ -838,11 +883,8 @@ public class LruBlockCache implements Bl
*/
SortedSet<String> getCachedFileNamesForTest() {
SortedSet<String> fileNames = new TreeSet<String>();
- for (String cacheKey : map.keySet()) {
- int sepIndex = cacheKey.indexOf(HFile.CACHE_KEY_SEPARATOR);
- if (sepIndex != -1) {
- fileNames.add(cacheKey.substring(0, sepIndex));
- }
+ for (BlockCacheKey cacheKey : map.keySet()) {
+ fileNames.add(cacheKey.getHfileName());
}
return fileNames;
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java Tue Feb 7 22:51:13 2012
@@ -23,8 +23,10 @@ import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
@@ -34,14 +36,14 @@ import org.apache.hadoop.hbase.io.hfile.
*/
public class SimpleBlockCache implements BlockCache {
private static class Ref extends SoftReference<Cacheable> {
- public String blockId;
- public Ref(String blockId, Cacheable block, ReferenceQueue q) {
+ public BlockCacheKey blockId;
+ public Ref(BlockCacheKey blockId, Cacheable block, ReferenceQueue q) {
super(block, q);
this.blockId = blockId;
}
}
- private Map<String,Ref> cache =
- new HashMap<String,Ref>();
+ private Map<BlockCacheKey,Ref> cache =
+ new HashMap<BlockCacheKey,Ref>();
private ReferenceQueue q = new ReferenceQueue();
public int dumps = 0;
@@ -64,31 +66,31 @@ public class SimpleBlockCache implements
/**
* @return the size
*/
- public synchronized int size() {
+ public synchronized long size() {
processQueue();
return cache.size();
}
- public synchronized Cacheable getBlock(String blockName, boolean caching) {
+ public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
processQueue(); // clear out some crap.
- Ref ref = cache.get(blockName);
+ Ref ref = cache.get(cacheKey);
if (ref == null)
return null;
return ref.get();
}
- public synchronized void cacheBlock(String blockName, Cacheable block) {
- cache.put(blockName, new Ref(blockName, block, q));
+ public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block) {
+ cache.put(cacheKey, new Ref(cacheKey, block, q));
}
- public synchronized void cacheBlock(String blockName, Cacheable block,
+ public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block,
boolean inMemory) {
- cache.put(blockName, new Ref(blockName, block, q));
+ cache.put(cacheKey, new Ref(cacheKey, block, q));
}
@Override
- public boolean evictBlock(String blockName) {
- return cache.remove(blockName) != null;
+ public boolean evictBlock(BlockCacheKey cacheKey) {
+ return cache.remove(cacheKey) != null;
}
public void shutdown() {
@@ -102,7 +104,30 @@ public class SimpleBlockCache implements
}
@Override
- public int evictBlocksByPrefix(String string) {
+ public long getFreeSize() {
+ // TODO: implement this if we ever actually use this block cache
+ return 0;
+ }
+
+ @Override
+ public long getCurrentSize() {
+ // TODO: implement this if we ever actually use this block cache
+ return 0;
+ }
+
+ @Override
+ public long getEvictedCount() {
+ // TODO: implement this if we ever actually use this block cache
+ return 0;
+ }
+
+ @Override
+ public int evictBlocksByHfileName(String string) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<BlockCacheColumnFamilySummary> getBlockCacheColumnFamilySummaries(Configuration conf) {
throw new UnsupportedOperationException();
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Feb 7 22:51:13 2012
@@ -49,6 +49,7 @@ import java.util.concurrent.CopyOnWriteA
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -84,7 +85,7 @@ import org.apache.hadoop.hbase.util.Thre
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
-import com.google.common.util.concurrent.NamingThreadFactory;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* HLog stores all the edits to the HStore. Its the hbase write-ahead-log
@@ -1656,8 +1657,10 @@ public class HLog implements Syncable {
conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);
boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
HashMap<byte[], Future> writeFutureResult = new HashMap<byte[], Future>();
- NamingThreadFactory f = new NamingThreadFactory(
- "SplitWriter-%1$d", Executors.defaultThreadFactory());
+ ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
+ tfb.setNameFormat("SplitWriter-%1$d");
+ tfb.setThreadFactory(Executors.defaultThreadFactory());
+ ThreadFactory f = tfb.build();
ThreadPoolExecutor threadPool = (ThreadPoolExecutor)Executors.newFixedThreadPool(logWriterThreads, f);
for (final byte [] region : splitLogsMap.keySet()) {
Callable splitter = createNewSplitter(rootDir, logWriters, splitLogsMap, region, fs, conf);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Tue Feb 7 22:51:13 2012
@@ -700,6 +700,62 @@ public class FSUtils {
}
/**
+ * Runs through the HBase rootdir and creates a reverse lookup map for
+ * table StoreFile names to the full Path.
+ * <br>
+ * Example...<br>
+ * Key = 3944417774205889744 <br>
+ * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ *
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @return Map keyed by StoreFile name with a value of the full Path.
+ * @throws IOException When scanning the directory fails.
+ */
+ public static Map<String, Path> getTableStoreFilePathMap(
+ final FileSystem fs, final Path hbaseRootDir)
+ throws IOException {
+ Map<String, Path> map = new HashMap<String, Path>();
+
+ // if this method looks similar to 'getTableFragmentation' that is because
+ // it was borrowed from it.
+
+ DirFilter df = new DirFilter(fs);
+ // presumes any directory under hbase.rootdir is a table
+ FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
+ for (FileStatus tableDir : tableDirs) {
+ // Skip the .log directory. All others should be tables. Inside a table,
+ // there are compaction.dir directories to skip. Otherwise, all else
+ // should be regions.
+ Path d = tableDir.getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
+ continue;
+ }
+ FileStatus[] regionDirs = fs.listStatus(d, df);
+ for (FileStatus regionDir : regionDirs) {
+ Path dd = regionDir.getPath();
+ if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
+ continue;
+ }
+ // else its a region name, now look in region for families
+ FileStatus[] familyDirs = fs.listStatus(dd, df);
+ for (FileStatus familyDir : familyDirs) {
+ Path family = familyDir.getPath();
+ // now in family, iterate over the StoreFiles and
+ // put in map
+ FileStatus[] familyStatus = fs.listStatus(family);
+ for (FileStatus sfStatus : familyStatus) {
+ Path sf = sfStatus.getPath();
+ map.put( sf.getName(), sf);
+ }
+
+ }
+ }
+ }
+ return map;
+ }
+
+ /**
* This function is to scan the root path of the file system to get the
* mapping between the region name and its best locality region server
*
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java?rev=1241679&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java Tue Feb 7 22:51:13 2012
@@ -0,0 +1,120 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+/**
+ * Tests the BlockCacheColumnFamilySummary class
+ *
+ */
+public class TestBlockCacheColumnFamilySummary {
+
+
+ /**
+ *
+ */
+ @Test
+ public void testEquals() {
+
+ BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary();
+ e1.setTable("table1");
+ e1.setColumnFamily("cf1");
+
+ BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary();
+ e2.setTable("table1");
+ e2.setColumnFamily("cf1");
+
+ assertEquals("bcse", e1, e2);
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testNotEquals() {
+
+ BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary();
+ e1.setTable("table1");
+ e1.setColumnFamily("cf1");
+
+ BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary();
+ e2.setTable("tablexxxxxx");
+ e2.setColumnFamily("cf1");
+
+ assertTrue("bcse", ! e1.equals(e2));
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testMapLookup() {
+
+ Map<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary> bcs =
+ new HashMap<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary>();
+
+ BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary("table1","cf1");
+
+ BlockCacheColumnFamilySummary lookup = bcs.get(e1);
+
+ if (lookup == null) {
+ lookup = BlockCacheColumnFamilySummary.create(e1);
+ bcs.put(e1,lookup);
+ lookup.incrementBlocks();
+ lookup.incrementHeapSize(100L);
+ }
+
+ BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary("table1","cf1");
+
+ BlockCacheColumnFamilySummary l2 = bcs.get(e2);
+ assertEquals("blocks",1,l2.getBlocks());
+ assertEquals("heap",100L,l2.getHeapSize());
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testMapEntry() {
+
+ Map<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary> bcs =
+ new HashMap<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary>();
+
+ BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary("table1","cf1");
+ bcs.put(e1, e1);
+
+ BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary("table1","cf1");
+ bcs.put(e2, e2);
+
+ BlockCacheColumnFamilySummary e3 = new BlockCacheColumnFamilySummary("table1","cf1");
+ bcs.put(e3, e3);
+
+ assertEquals("mapSize",1,bcs.size());
+ }
+
+}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Tue Feb 7 22:51:13 2012
@@ -152,7 +152,8 @@ public class TestCacheOnWrite {
@After
public void tearDown() {
- blockCache.evictBlocksByPrefix("");
+ cacheConf = new CacheConfig(conf);
+ blockCache = cacheConf.getBlockCache();
}
@Test
@@ -181,7 +182,7 @@ public class TestCacheOnWrite {
// Flags: don't cache the block, use pread, this is not a compaction.
HFileBlock block = reader.readBlock(offset, onDiskSize, false, true,
false);
- String blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset);
+ BlockCacheKey blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset);
boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
assertEquals(testName + " " + block, shouldBeCached, isCached);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java Tue Feb 7 22:51:13 2012
@@ -19,13 +19,10 @@
*/
package org.apache.hadoop.hbase.io.hfile;
-import java.nio.ByteBuffer;
+import junit.framework.TestCase;
-import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
-import junit.framework.TestCase;
-
public class TestCachedBlockQueue extends TestCase {
public void testQueue() throws Exception {
@@ -61,16 +58,9 @@ public class TestCachedBlockQueue extend
assertEquals(queue.heapSize(), expectedSize);
- org.apache.hadoop.hbase.io.hfile.CachedBlock [] blocks = queue.get();
- assertEquals(blocks[0].getName(), "cb1");
- assertEquals(blocks[1].getName(), "cb2");
- assertEquals(blocks[2].getName(), "cb3");
- assertEquals(blocks[3].getName(), "cb4");
- assertEquals(blocks[4].getName(), "cb5");
- assertEquals(blocks[5].getName(), "cb6");
- assertEquals(blocks[6].getName(), "cb7");
- assertEquals(blocks[7].getName(), "cb8");
-
+ for (int i = 1; i <= 8; i++) {
+ assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb"+i);
+ }
}
public void testQueueSmallBlockEdgeCase() throws Exception {
@@ -113,23 +103,15 @@ public class TestCachedBlockQueue extend
assertEquals(queue.heapSize(), expectedSize);
- org.apache.hadoop.hbase.io.hfile.CachedBlock [] blocks = queue.get();
- assertEquals(blocks[0].getName(), "cb0");
- assertEquals(blocks[1].getName(), "cb1");
- assertEquals(blocks[2].getName(), "cb2");
- assertEquals(blocks[3].getName(), "cb3");
- assertEquals(blocks[4].getName(), "cb4");
- assertEquals(blocks[5].getName(), "cb5");
- assertEquals(blocks[6].getName(), "cb6");
- assertEquals(blocks[7].getName(), "cb7");
- assertEquals(blocks[8].getName(), "cb8");
-
+ for (int i = 0; i <= 8; i++) {
+ assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb"+i);
+ }
}
private static class CachedBlock extends org.apache.hadoop.hbase.io.hfile.CachedBlock
{
public CachedBlock(final long heapSize, String name, long accessTime) {
- super(name,
+ super(new BlockCacheKey(name, 0),
new Cacheable() {
@Override
public long heapSize() {
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Tue Feb 7 22:51:13 2012
@@ -468,8 +468,7 @@ public class TestHFileBlockIndex {
LOG.info("Index block size: " + indexBlockSize + ", compression: "
+ compr);
// Evict all blocks that were cached-on-write by the previous invocation.
- blockCache.evictBlocksByPrefix(hfilePath.getName()
- + HFile.CACHE_KEY_SEPARATOR);
+ blockCache.evictBlocksByHfileName(hfilePath.getName());
conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
Set<String> keyStrSet = new HashSet<String>();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1241679&r1=1241678&r2=1241679&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Tue Feb 7 22:51:13 2012
@@ -79,7 +79,7 @@ public class TestLruBlockCache {
// Add all the blocks
for (CachedItem block : blocks) {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
}
// Let the eviction run
@@ -109,12 +109,12 @@ public class TestLruBlockCache {
// Confirm empty
for (CachedItem block : blocks) {
- assertTrue(cache.getBlock(block.blockName, true) == null);
+ assertTrue(cache.getBlock(block.cacheKey, true) == null);
}
// Add blocks
for (CachedItem block : blocks) {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
expectedCacheSize += block.cacheBlockHeapSize();
}
@@ -123,7 +123,7 @@ public class TestLruBlockCache {
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.blockName, true);
+ HeapSize buf = cache.getBlock(block.cacheKey, true);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -131,7 +131,7 @@ public class TestLruBlockCache {
// Re-add same blocks and ensure nothing has changed
for (CachedItem block : blocks) {
try {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
assertTrue("Cache should not allow re-caching a block", false);
} catch(RuntimeException re) {
// expected
@@ -143,7 +143,7 @@ public class TestLruBlockCache {
// Check if all blocks are properly cached and retrieved
for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.blockName, true);
+ HeapSize buf = cache.getBlock(block.cacheKey, true);
assertTrue(buf != null);
assertEquals(buf.heapSize(), block.heapSize());
}
@@ -166,7 +166,7 @@ public class TestLruBlockCache {
// Add all the blocks
for (CachedItem block : blocks) {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
expectedCacheSize += block.cacheBlockHeapSize();
}
@@ -185,10 +185,10 @@ public class TestLruBlockCache {
(maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
// All blocks except block 0 and 1 should be in the cache
- assertTrue(cache.getBlock(blocks[0].blockName, true) == null);
- assertTrue(cache.getBlock(blocks[1].blockName, true) == null);
+ assertTrue(cache.getBlock(blocks[0].cacheKey, true) == null);
+ assertTrue(cache.getBlock(blocks[1].cacheKey, true) == null);
for(int i=2;i<blocks.length;i++) {
- assertEquals(cache.getBlock(blocks[i].blockName, true),
+ assertEquals(cache.getBlock(blocks[i].cacheKey, true),
blocks[i]);
}
}
@@ -208,14 +208,14 @@ public class TestLruBlockCache {
// Add and get the multi blocks
for (CachedItem block : multiBlocks) {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
expectedCacheSize += block.cacheBlockHeapSize();
- assertEquals(cache.getBlock(block.blockName, true), block);
+ assertEquals(cache.getBlock(block.cacheKey, true), block);
}
// Add the single blocks (no get)
for (CachedItem block : singleBlocks) {
- cache.cacheBlock(block.blockName, block);
+ cache.cacheBlock(block.cacheKey, block);
expectedCacheSize += block.heapSize();
}
@@ -240,14 +240,14 @@ public class TestLruBlockCache {
// This test makes multi go barely over its limit, in-memory
// empty, and the rest in single. Two single evictions and
// one multi eviction expected.
- assertTrue(cache.getBlock(singleBlocks[0].blockName, true) == null);
- assertTrue(cache.getBlock(multiBlocks[0].blockName, true) == null);
+ assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true) == null);
+ assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true) == null);
// And all others to be cached
for(int i=1;i<4;i++) {
- assertEquals(cache.getBlock(singleBlocks[i].blockName, true),
+ assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true),
singleBlocks[i]);
- assertEquals(cache.getBlock(multiBlocks[i].blockName, true),
+ assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true),
multiBlocks[i]);
}
}
@@ -279,16 +279,16 @@ public class TestLruBlockCache {
for(int i=0;i<3;i++) {
// Just add single blocks
- cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
// Add and get multi blocks
- cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
- cache.getBlock(multiBlocks[i].blockName, true);
+ cache.getBlock(multiBlocks[i].cacheKey, true);
// Add memory blocks as such
- cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
+ cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
}
@@ -300,77 +300,77 @@ public class TestLruBlockCache {
assertEquals(expectedCacheSize, cache.heapSize());
// Insert a single block, oldest single should be evicted
- cache.cacheBlock(singleBlocks[3].blockName, singleBlocks[3]);
+ cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]);
// Single eviction, one thing evicted
assertEquals(1, cache.getEvictionCount());
assertEquals(1, cache.getEvictedCount());
// Verify oldest single block is the one evicted
- assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
// Change the oldest remaining single block to a multi
- cache.getBlock(singleBlocks[1].blockName, true);
+ cache.getBlock(singleBlocks[1].cacheKey, true);
// Insert another single block
- cache.cacheBlock(singleBlocks[4].blockName, singleBlocks[4]);
+ cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
// Two evictions, two evicted.
assertEquals(2, cache.getEvictionCount());
assertEquals(2, cache.getEvictedCount());
// Oldest multi block should be evicted now
- assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
// Insert another memory block
- cache.cacheBlock(memoryBlocks[3].blockName, memoryBlocks[3], true);
+ cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
// Three evictions, three evicted.
assertEquals(3, cache.getEvictionCount());
assertEquals(3, cache.getEvictedCount());
// Oldest memory block should be evicted now
- assertEquals(null, cache.getBlock(memoryBlocks[0].blockName, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true));
// Add a block that is twice as big (should force two evictions)
CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
- cache.cacheBlock(bigBlocks[0].blockName, bigBlocks[0]);
+ cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]);
// Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
assertEquals(4, cache.getEvictionCount());
assertEquals(6, cache.getEvictedCount());
// Expect three remaining singles to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[2].blockName, true));
- assertEquals(null, cache.getBlock(singleBlocks[3].blockName, true));
- assertEquals(null, cache.getBlock(singleBlocks[4].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true));
// Make the big block a multi block
- cache.getBlock(bigBlocks[0].blockName, true);
+ cache.getBlock(bigBlocks[0].cacheKey, true);
// Cache another single big block
- cache.cacheBlock(bigBlocks[1].blockName, bigBlocks[1]);
+ cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
// Five evictions, nine evicted (3 new)
assertEquals(5, cache.getEvictionCount());
assertEquals(9, cache.getEvictedCount());
// Expect three remaining multis to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
- assertEquals(null, cache.getBlock(multiBlocks[2].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true));
// Cache a big memory block
- cache.cacheBlock(bigBlocks[2].blockName, bigBlocks[2], true);
+ cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
// Six evictions, twelve evicted (3 new)
assertEquals(6, cache.getEvictionCount());
assertEquals(12, cache.getEvictedCount());
// Expect three remaining in-memory to be evicted
- assertEquals(null, cache.getBlock(memoryBlocks[1].blockName, true));
- assertEquals(null, cache.getBlock(memoryBlocks[2].blockName, true));
- assertEquals(null, cache.getBlock(memoryBlocks[3].blockName, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true));
}
@@ -397,13 +397,13 @@ public class TestLruBlockCache {
// Add 5 multi blocks
for (CachedItem block : multiBlocks) {
- cache.cacheBlock(block.blockName, block);
- cache.getBlock(block.blockName, true);
+ cache.cacheBlock(block.cacheKey, block);
+ cache.getBlock(block.cacheKey, true);
}
// Add 5 single blocks
for(int i=0;i<5;i++) {
- cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
}
// An eviction ran
@@ -413,10 +413,10 @@ public class TestLruBlockCache {
assertEquals(4, cache.getEvictedCount());
// Should have been taken off equally from single and multi
- assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
- assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
- assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
// Let's keep "scanning" by adding single blocks. From here on we only
// expect evictions from the single bucket.
@@ -426,7 +426,7 @@ public class TestLruBlockCache {
// 12 more evicted.
for(int i=5;i<18;i++) {
- cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
}
// 4 total evictions, 16 total evicted
@@ -463,14 +463,14 @@ public class TestLruBlockCache {
for(int i=0;i<10;i++) {
// Just add single blocks
- cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
// Add and get multi blocks
- cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
- cache.getBlock(multiBlocks[i].blockName, true);
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ cache.getBlock(multiBlocks[i].cacheKey, true);
// Add memory blocks as such
- cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
+ cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
}
// Do not expect any evictions yet
@@ -487,19 +487,16 @@ public class TestLruBlockCache {
// And the oldest 5 blocks from each category should be gone
for(int i=0;i<5;i++) {
- assertEquals(null, cache.getBlock(singleBlocks[i].blockName, true));
- assertEquals(null, cache.getBlock(multiBlocks[i].blockName, true));
- assertEquals(null, cache.getBlock(memoryBlocks[i].blockName, true));
+ assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true));
+ assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true));
}
// And the newest 5 blocks should still be accessible
for(int i=5;i<10;i++) {
- assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].blockName,
- true));
- assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].blockName,
- true));
- assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].blockName,
- true));
+ assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true));
+ assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true));
+ assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true));
}
}
@@ -550,11 +547,11 @@ public class TestLruBlockCache {
}
private static class CachedItem implements Cacheable {
- String blockName;
+ BlockCacheKey cacheKey;
int size;
CachedItem(String blockName, int size) {
- this.blockName = blockName;
+ this.cacheKey = new BlockCacheKey(blockName, 0);
this.size = size;
}
@@ -567,7 +564,7 @@ public class TestLruBlockCache {
/** Size of the cache block holding this item. Used for verification. */
public long cacheBlockHeapSize() {
return CachedBlock.PER_BLOCK_OVERHEAD
- + ClassSize.align(2 * blockName.length())
+ + ClassSize.align(cacheKey.heapSize())
+ ClassSize.align(size);
}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java?rev=1241679&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java Tue Feb 7 22:51:13 2012
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests the block cache summary functionality in StoreFile,
+ * which contains the BlockCache
+ *
+ */
+public class TestStoreFileBlockCacheSummary {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final String TEST_TABLE = "testTable";
+ private static final String TEST_TABLE2 = "testTable2";
+ private static final String TEST_CF = "testFamily";
+ private static byte [] FAMILY = Bytes.toBytes(TEST_CF);
+ private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
+ private static byte [] VALUE = Bytes.toBytes("testValue");
+
+ private final int TOTAL_ROWS = 4;
+
+ /**
+ * @throws java.lang.Exception exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster();
+ }
+
+ /**
+ * @throws java.lang.Exception exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+
+ private Put createPut(byte[] family, String row) {
+ Put put = new Put( Bytes.toBytes(row));
+ put.add(family, QUALIFIER, VALUE);
+ return put;
+ }
+
+ /**
+ * This test inserts data into multiple tables and then reads both tables to ensure
+ * they are in the block cache.
+ *
+ * @throws Exception exception
+ */
+ @Test
+ public void testBlockCacheSummary() throws Exception {
+ HTable ht = TEST_UTIL.createTable(Bytes.toBytes(TEST_TABLE), FAMILY);
+ addRows(ht, FAMILY);
+
+ HTable ht2 = TEST_UTIL.createTable(Bytes.toBytes(TEST_TABLE2), FAMILY);
+ addRows(ht2, FAMILY);
+
+ TEST_UTIL.flush();
+
+ scan(ht, FAMILY);
+ scan(ht2, FAMILY);
+
+ BlockCache bc =
+ new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache();
+ List<BlockCacheColumnFamilySummary> bcs =
+ bc.getBlockCacheColumnFamilySummaries(TEST_UTIL.getConfiguration());
+ LOG.info("blockCacheSummary: " + bcs);
+
+ assertEquals("blockCache summary has entries", 3, bcs.size());
+
+ BlockCacheColumnFamilySummary e = bcs.get(0);
+ assertEquals("table", "-ROOT-", e.getTable());
+ assertEquals("cf", "info", e.getColumnFamily());
+
+ e = bcs.get(1);
+ assertEquals("table", TEST_TABLE, e.getTable());
+ assertEquals("cf", TEST_CF, e.getColumnFamily());
+
+ e = bcs.get(2);
+ assertEquals("table", TEST_TABLE2, e.getTable());
+ assertEquals("cf", TEST_CF, e.getColumnFamily());
+
+ ht.close();
+ ht2.close();
+ }
+
+ private void addRows(HTable ht, byte[] family) throws IOException {
+
+ for (int i = 0; i < TOTAL_ROWS;i++) {
+ ht.put(createPut(family, "row" + i));
+ }
+ }
+
+ private void scan(HTable ht, byte[] family) throws IOException {
+ Scan scan = new Scan();
+ scan.addColumn(family, QUALIFIER);
+
+ int count = 0;
+ for(@SuppressWarnings("unused") Result result : ht.getScanner(scan)) {
+ count++;
+ }
+ if (TOTAL_ROWS != count) {
+ throw new IOException("Incorrect number of rows!");
+ }
+ }
+
+}