You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2017/09/08 19:45:29 UTC
hbase git commit: HBASE-18641 Include block content verification
logic used in lruCache in bucketCache - revert due to test failure
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 3f831ebdc -> e177f71fd
HBASE-18641 Include block content verification logic used in lruCache in bucketCache - revert due to test failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e177f71f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e177f71f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e177f71f
Branch: refs/heads/branch-1.4
Commit: e177f71fd13f00faa104172a0a42bb6a7f5f9040
Parents: 3f831eb
Author: tedyu <yu...@gmail.com>
Authored: Fri Sep 8 12:45:24 2017 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Sep 8 12:45:24 2017 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/hfile/BlockCacheUtil.java | 11 -----------
.../hadoop/hbase/io/hfile/LruBlockCache.java | 13 +++++++++++--
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 17 ++---------------
3 files changed, 13 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
index a3d46ed..e5bb83b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.io.hfile;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -27,7 +26,6 @@ import java.util.concurrent.ConcurrentSkipListSet;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram;
-import org.apache.hadoop.hbase.util.Bytes;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.JsonMappingException;
@@ -173,15 +171,6 @@ public class BlockCacheUtil {
return cbsbf;
}
- public static int compareCacheBlock(Cacheable left, Cacheable right) {
- ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
- left.serialize(l);
- ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
- right.serialize(r);
- return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
- r.array(), r.arrayOffset(), r.limit());
- }
-
/**
* Use one of these to keep a running account of cached blocks by file. Throw it away when done.
* This is different than metrics in that it is stats on current state of a cache.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 69fea76..3fbe44b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -373,7 +373,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
LruCachedBlock cb = map.get(cacheKey);
if (cb != null) {
// compare the contents, if they are not equal, we are in big trouble
- if (BlockCacheUtil.compareCacheBlock(buf, cb.getBuffer()) != 0) {
+ if (compare(buf, cb.getBuffer()) != 0) {
throw new RuntimeException("Cached block contents differ, which should not have happened."
+ "cacheKey:" + cacheKey);
}
@@ -433,6 +433,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
}
}
+ private int compare(Cacheable left, Cacheable right) {
+ ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
+ left.serialize(l);
+ ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
+ right.serialize(r);
+ return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
+ r.array(), r.arrayOffset(), r.limit());
+ }
+
/**
* Cache the block with the specified name and buffer.
* <p>
@@ -568,7 +577,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
boolean wait = getCurrentSize() < acceptableSize();
boolean inMemory = block.getPriority() == BlockPriority.MEMORY;
((BucketCache)victimHandler).cacheBlockWithWait(block.getCacheKey(), block.getBuffer(),
- inMemory, true, wait);
+ inMemory, wait);
} else {
victimHandler.cacheBlock(block.getCacheKey(), block.getBuffer());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index b8d7453..6d50bef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -402,7 +402,7 @@ public class BucketCache implements BlockCache, HeapSize {
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory,
final boolean cacheDataInL1) {
- cacheBlockWithWait(cacheKey, cachedItem, inMemory, cacheDataInL1, wait_when_cache);
+ cacheBlockWithWait(cacheKey, cachedItem, inMemory, wait_when_cache);
}
/**
@@ -413,26 +413,13 @@ public class BucketCache implements BlockCache, HeapSize {
* @param wait if true, blocking wait when queue is full
*/
public void cacheBlockWithWait(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory,
- boolean cacheDataInL1, boolean wait) {
+ boolean wait) {
if (LOG.isTraceEnabled()) LOG.trace("Caching key=" + cacheKey + ", item=" + cachedItem);
if (!cacheEnabled) {
return;
}
if (backingMap.containsKey(cacheKey)) {
- /*
- * Compare already cached block only if lruBlockCache is not used to cache data blocks
- */
- if (!cacheDataInL1) {
- Cacheable existingBlock = getBlock(cacheKey, false, false, false);
- if (BlockCacheUtil.compareCacheBlock(cachedItem, existingBlock) != 0) {
- throw new RuntimeException("Cached block contents differ, which should not have happened."
- + "cacheKey:" + cacheKey);
- }
- }
- String msg = "Caching an already cached block: " + cacheKey;
- msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
- LOG.warn(msg);
return;
}