You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/05/19 17:53:07 UTC
svn commit: r1595938 - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/
Author: stack
Date: Mon May 19 15:53:06 2014
New Revision: 1595938
URL: http://svn.apache.org/r1595938
Log:
HBASE-11007 BLOCKCACHE in schema descriptor seems not aptly named
Modified:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=1595938&r1=1595937&r2=1595938&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java Mon May 19 15:53:06 2014
@@ -77,6 +77,12 @@ public class HColumnDescriptor implement
"ENCODE_ON_DISK";
public static final String DATA_BLOCK_ENCODING =
"DATA_BLOCK_ENCODING";
+ /**
+ * Key for the BLOCKCACHE attribute.
+ * A more exact name would be CACHE_DATA_ON_READ because this flag sets whether or not we
+ * cache DATA blocks. We always cache INDEX and BLOOM blocks; caching these blocks cannot be
+ * disabled.
+ */
public static final String BLOCKCACHE = "BLOCKCACHE";
public static final String CACHE_DATA_ON_WRITE = "CACHE_DATA_ON_WRITE";
public static final String CACHE_INDEX_ON_WRITE = "CACHE_INDEX_ON_WRITE";
@@ -792,7 +798,8 @@ public class HColumnDescriptor implement
}
/**
- * @return True if MapFile blocks should be cached.
+ * @return True if hfile DATA type blocks should be cached (You cannot disable caching of INDEX
+ * and BLOOM type blocks).
*/
public boolean isBlockCacheEnabled() {
String value = getValue(BLOCKCACHE);
@@ -802,7 +809,8 @@ public class HColumnDescriptor implement
}
/**
- * @param blockCacheEnabled True if MapFile blocks should be cached.
+ * @param blockCacheEnabled True if hfile DATA type blocks should be cached (We always cache
+ * INDEX and BLOOM blocks; you cannot turn this off).
* @return this (for chained invocation)
*/
public HColumnDescriptor setBlockCacheEnabled(boolean blockCacheEnabled) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java?rev=1595938&r1=1595937&r2=1595938&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java Mon May 19 15:53:06 2014
@@ -161,7 +161,9 @@ public class CacheConfig {
/**
* Whether blocks should be cached on read (default is on if there is a
- * cache but this can be turned off on a per-family or per-request basis)
+ * cache but this can be turned off on a per-family or per-request basis).
+ * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types.
+ * This cannot be disabled.
*/
private boolean cacheDataOnRead;
@@ -232,7 +234,8 @@ public class CacheConfig {
* Create a block cache configuration with the specified cache and
* configuration parameters.
* @param blockCache reference to block cache, null if completely disabled
- * @param cacheDataOnRead whether data blocks should be cached on read
+ * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX
+ * blocks and BLOOM blocks; this cannot be disabled).
* @param inMemory whether blocks should be flagged as in-memory
* @param cacheDataOnWrite whether data blocks should be cached on write
* @param cacheIndexesOnWrite whether index blocks should be cached on write
@@ -283,7 +286,8 @@ public class CacheConfig {
}
/**
- * Returns whether the blocks of this HFile should be cached on read or not.
+ * Returns whether the DATA blocks of this HFile should be cached on read or not (we always
+ * cache the meta blocks, the INDEX and BLOOM blocks).
* @return true if blocks should be cached on read, false if not
*/
public boolean shouldCacheDataOnRead() {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java?rev=1595938&r1=1595937&r2=1595938&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java Mon May 19 15:53:06 2014
@@ -87,6 +87,12 @@ public class CacheStats {
this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
}
+ @Override
+ public String toString() {
+ return "hitCount=" + this.hitCount + ", hitCachingCount=" + this.hitCachingCount +
+ ", missCount=" + this.missCount;
+ }
+
public void miss(boolean caching) {
missCount.incrementAndGet();
if (caching) missCachingCount.incrementAndGet();
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java?rev=1595938&r1=1595937&r2=1595938&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java Mon May 19 15:53:06 2014
@@ -16,11 +16,12 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.List;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -29,10 +30,10 @@ import org.apache.hadoop.hbase.client.Ge
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
-import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -43,11 +44,14 @@ import org.junit.runners.Parameterized.P
* Make sure we always cache important block types, such as index blocks, as
* long as we have a block cache, even though block caching might be disabled
* for the column family.
+ *
+ * <p>TODO: This test writes a lot of data and only tests the most basic of metrics. Cache stats
+ * need to reveal more about what is being cached whether DATA or INDEX blocks and then we could
+ * do more verification in this test.
*/
@Category(MediumTests.class)
@RunWith(Parameterized.class)
public class TestForceCacheImportantBlocks {
-
private final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static final String TABLE = "myTable";
@@ -68,6 +72,8 @@ public class TestForceCacheImportantBloc
Compression.Algorithm.GZ;
private static final BloomType BLOOM_TYPE = BloomType.ROW;
+ @SuppressWarnings("unused")
+ // Currently unused.
private final int hfileVersion;
private final boolean cfCacheEnabled;
@@ -76,7 +82,7 @@ public class TestForceCacheImportantBloc
// HFile versions
return Arrays.asList(new Object[][] {
new Object[] { new Integer(2), false },
- new Object[] { new Integer(2), true }
+ new Object[] { new Integer(3), true }
});
}
@@ -84,42 +90,42 @@ public class TestForceCacheImportantBloc
boolean cfCacheEnabled) {
this.hfileVersion = hfileVersion;
this.cfCacheEnabled = cfCacheEnabled;
- TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY,
- hfileVersion);
+ TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, hfileVersion);
+ }
+
+ @Before
+ public void setup() {
+ // Make sure we make a new one each time.
+ CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ HFile.dataBlockReadCnt.set(0);
}
@Test
public void testCacheBlocks() throws IOException {
// Set index block size to be the same as normal block size.
- TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY,
- BLOCK_SIZE);
-
-
- HColumnDescriptor hcd =
- new HColumnDescriptor(Bytes.toBytes(CF))
- .setMaxVersions(MAX_VERSIONS)
- .setCompressionType(COMPRESSION_ALGORITHM)
- .setBloomFilterType(BLOOM_TYPE);
+ TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE);
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS).
+ setCompressionType(COMPRESSION_ALGORITHM).
+ setBloomFilterType(BLOOM_TYPE);
hcd.setBlocksize(BLOCK_SIZE);
hcd.setBlockCacheEnabled(cfCacheEnabled);
HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd);
writeTestData(region);
-
- for (int i = 0; i < NUM_ROWS; ++i) {
- Get get = new Get(Bytes.toBytes("row" + i));
- region.get(get);
- }
-
- List<BlockCategory> importantBlockCategories =
- new ArrayList<BlockCategory>();
- importantBlockCategories.add(BlockCategory.BLOOM);
- if (hfileVersion == 2) {
- // We only have index blocks for HFile v2.
- importantBlockCategories.add(BlockCategory.INDEX);
- }
+ CacheStats stats =
+ region.getStores().get(hcd.getName()).getCacheConfig().getBlockCache().getStats();
+ assertEquals(0, stats.getHitCount());
+ assertEquals(0, HFile.dataBlockReadCnt.get());
+ // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss
+ // count should go up. Otherwise, all should be cached and the miss count should not rise.
+ region.get(new Get(Bytes.toBytes("row" + 0)));
+ assertTrue(stats.getHitCount() > 0);
+ assertTrue(HFile.dataBlockReadCnt.get() > 0);
+ long missCount = stats.getMissCount();
+ region.get(new Get(Bytes.toBytes("row" + 0)));
+ if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount());
+ else assertTrue(stats.getMissCount() > missCount);
}
-
private void writeTestData(HRegion region) throws IOException {
for (int i = 0; i < NUM_ROWS; ++i) {
Put put = new Put(Bytes.toBytes("row" + i));
@@ -135,6 +141,4 @@ public class TestForceCacheImportantBloc
}
}
}
-
-}
-
+}
\ No newline at end of file