[bookkeeper] branch master updated: Fixed DbStorage write cache segment index computation

[mm...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git


The following commit(s) were added to refs/heads/master by this push:
     new 6edbdd9  Fixed DbStorage write cache segment index computation
6edbdd9 is described below

commit 6edbdd905dbb1a20a7e9282047c6a255d1e9f527
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Tue Mar 20 17:02:39 2018 -0700

    Fixed DbStorage write cache segment index computation
    
    When the write cache was merged from `yahoo-4.3` branch into master, an improvement went in around computing the the write cache segment index (doing shift instead of division) and that introduced a problem.
    
    The write cache is broken into multiple segments (since buffers in JVM can only be at most 2GB). By default we use 1GB segments (or smaller if the write cache is overall smaller).
    
    Because the shift count was wrong, the entries in the first segment were being ovrridden by entries in the 2nd segment (entries in the other segments were all fine).
    
    That would lead to write some garbage in the entry log and subsequent read failures.
    
    Author: Matteo Merli <mm...@apache.org>
    
    Reviewers: Sijie Guo <si...@apache.org>
    
    This closes #1279 from merlimat/fix-write-cache-segment-index
---
 .../bookkeeper/bookie/storage/ldb/WriteCache.java  |  2 +-
 .../bookie/storage/ldb/WriteCacheTest.java         | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
index 1544b6d..08ffe67 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
@@ -94,7 +94,7 @@ public class WriteCache implements Closeable {
         this.maxCacheSize = maxCacheSize;
         this.maxSegmentSize = (int) maxSegmentSize;
         this.segmentOffsetMask = maxSegmentSize - 1;
-        this.segmentOffsetBits = 64 - Long.numberOfLeadingZeros(maxSegmentSize);
+        this.segmentOffsetBits = 63 - Long.numberOfLeadingZeros(maxSegmentSize);
 
         this.segmentsCount = 1 + (int) (maxCacheSize / maxSegmentSize);
 
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
index 75feb32..f8b2bba 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
@@ -30,6 +30,7 @@ import io.netty.buffer.ByteBufUtil;
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.buffer.Unpooled;
 
+import java.nio.charset.Charset;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
@@ -261,4 +262,25 @@ public class WriteCacheTest {
         cache.close();
     }
 
+    @Test
+    public void testWriteReadsInMultipleSegments() {
+        // Create cache with max size 4 KB and each segment is 128 bytes
+        WriteCache cache = new WriteCache(4 * 1024, 128);
+
+        for (int i = 0; i < 48; i++) {
+            boolean inserted = cache.put(1, i, Unpooled.wrappedBuffer(("test-" + i).getBytes()));
+            assertTrue(inserted);
+        }
+
+        assertEquals(48, cache.count());
+
+        for (int i = 0; i < 48; i++) {
+            ByteBuf b = cache.get(1, i);
+
+            assertEquals("test-" + i, b.toString(Charset.forName("UTF-8")));
+        }
+
+        cache.close();
+    }
+
 }

-- 
To stop receiving notification emails like this one, please contact
mmerli@apache.org.