You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/07/16 02:35:37 UTC

[GitHub] sijie closed pull request #1268: (WIP) Enable bytebuf leak detection in tests

sijie closed pull request #1268: (WIP) Enable bytebuf leak detection in tests
URL: https://github.com/apache/bookkeeper/pull/1268
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannel.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannel.java
index 05a20e549..105de2406 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannel.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannel.java
@@ -22,8 +22,8 @@
 package org.apache.bookkeeper.bookie;
 
 import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
 
+import io.netty.buffer.Unpooled;
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -78,7 +78,7 @@ public BufferedChannel(FileChannel fc, int writeCapacity, int readCapacity, long
         this.writeCapacity = writeCapacity;
         this.position = new AtomicLong(fc.position());
         this.writeBufferStartPosition.set(position.get());
-        this.writeBuffer = ByteBufAllocator.DEFAULT.directBuffer(writeCapacity);
+        this.writeBuffer = Unpooled.directBuffer(writeCapacity);
         this.unpersistedBytes = new AtomicLong(0);
         this.unpersistedBytesBound = unpersistedBytesBound;
     }
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestMaxSizeWorkersQueue.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestMaxSizeWorkersQueue.java
index cc70e024a..3ad0895c4 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestMaxSizeWorkersQueue.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestMaxSizeWorkersQueue.java
@@ -23,7 +23,9 @@
 
 import com.google.common.collect.Lists;
 
-import java.util.Enumeration;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import io.netty.buffer.ByteBuf;
+import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
@@ -34,20 +36,26 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.BookieException;
 import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
-import org.apache.bookkeeper.client.AsyncCallback.ReadCallback;
+import org.apache.bookkeeper.client.BKException.Code;
 import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
 import org.junit.Test;
 
 /**
  * Test the maximum size of a worker queue.
  */
+@Slf4j
 public class TestMaxSizeWorkersQueue extends BookKeeperClusterTestCase {
     DigestType digestType = DigestType.CRC32;
 
     public TestMaxSizeWorkersQueue() {
-        super(1);
+        super(0);
 
         baseConf.setNumReadWorkerThreads(1);
         baseConf.setNumAddWorkerThreads(1);
@@ -55,54 +63,101 @@ public TestMaxSizeWorkersQueue() {
         // Configure very small queue sizes
         baseConf.setMaxPendingReadRequestPerThread(1);
         baseConf.setMaxPendingAddRequestPerThread(1);
+
+        baseClientConf.setReadEntryTimeout(Integer.MAX_VALUE);
     }
 
     @Test
     public void testReadRejected() throws Exception {
-        LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
+        final CountDownLatch readLatch = new CountDownLatch(1);
+        final CountDownLatch readReceivedLatch = new CountDownLatch(1);
+
+        ServerConfiguration conf = newServerConfiguration();
+        bsConfs.add(conf);
+        bs.add(startBookie(conf, () -> {
+            try {
+                return new Bookie(conf) {
+                    @Override
+                    public ByteBuf readEntry(long ledgerId, long entryId) throws IOException, NoLedgerException {
+                        readReceivedLatch.countDown();
+                        try {
+                            readLatch.await();
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                            log.warn("Interrupted on reading entry {}-{}",
+                                ledgerId, entryId, e);
+                        }
+                        return super.readEntry(ledgerId, entryId);
+                    }
+                };
+            } catch (IOException | BookieException e) {
+                log.warn("Fail to construct bookie", e);
+                throw new UncheckedExecutionException(e);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                log.warn("Interrupt on constructing bookie", e);
+                throw new UncheckedExecutionException(e);
+            }
+        }));
+
+        @Cleanup LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
         byte[] content = new byte[100];
 
-        final int n = 1000;
+        final int n = 3;
         // Write few entries
         for (int i = 0; i < n; i++) {
             lh.addEntry(content);
         }
 
         // Read asynchronously:
-        // - 1st read must always succeed
-        // - Subsequent reads may fail, depending on timing
-        // - At least few, we expect to fail with TooManyRequestException
-        final CountDownLatch counter = new CountDownLatch(2);
+        // - 1st read is running (but blocking by testing logic) at the read thread
+        // - 2nd read is pending at the worker queue
+        // - 3rd read will be rejected with too many requests exception
 
         final AtomicInteger rcFirstReadOperation = new AtomicInteger();
+        final CountDownLatch firstReadLatch = new CountDownLatch(1);
 
-        lh.asyncReadEntries(0, 0, new ReadCallback() {
-            @Override
-            public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
-                rcFirstReadOperation.set(rc);
-                counter.countDown();
-            }
+        lh.asyncReadEntries(0, 0, (rc, lh1, seq, ctx) -> {
+            rcFirstReadOperation.set(rc);
+            firstReadLatch.countDown();
         }, lh);
+        readReceivedLatch.await();
 
         final AtomicInteger rcSecondReadOperation = new AtomicInteger();
+        final CountDownLatch secondReadLatch = new CountDownLatch(1);
 
-        lh.asyncReadEntries(0, n - 1, new ReadCallback() {
-            @Override
-            public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
-                rcSecondReadOperation.set(rc);
-                counter.countDown();
-            }
+        lh.asyncReadEntries(1, 1, (rc, lh12, seq, ctx) -> {
+            rcSecondReadOperation.set(rc);
+            secondReadLatch.countDown();
         }, lh);
 
-        counter.await();
+        final AtomicInteger rcThirdReadOperation = new AtomicInteger();
+        final CountDownLatch thirdReadLatch = new CountDownLatch(1);
 
+        lh.asyncReadEntries(2, 2, (rc, lh13, seq, ctx) -> {
+            rcThirdReadOperation.set(rc);
+            thirdReadLatch.countDown();
+        }, lh);
+
+        // third read should fail with too many requests exception
+        thirdReadLatch.await();
+        assertEquals(BKException.Code.TooManyRequestsException, rcThirdReadOperation.get());
+
+        // let first read go through
+        readLatch.countDown();
+        firstReadLatch.await();
         assertEquals(BKException.Code.OK, rcFirstReadOperation.get());
-        assertEquals(BKException.Code.TooManyRequestsException, rcSecondReadOperation.get());
+        secondReadLatch.await();
+        assertEquals(BKException.Code.OK, rcSecondReadOperation.get());
     }
 
     @Test
     public void testAddRejected() throws Exception {
-        LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
+        ServerConfiguration conf = newServerConfiguration();
+        bsConfs.add(conf);
+        bs.add(startBookie(conf));
+
+        @Cleanup LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
         byte[] content = new byte[100];
 
         final int n = 1000;
@@ -111,6 +166,7 @@ public void testAddRejected() throws Exception {
         // because when we get the TooManyRequestException, the client will try to form a new ensemble and that
         // operation will fail since we only have 1 bookie available
         final CountDownLatch counter = new CountDownLatch(n);
+        final AtomicBoolean receivedException = new AtomicBoolean(false);
         final AtomicBoolean receivedTooManyRequestsException = new AtomicBoolean();
 
         // Write few entries
@@ -118,6 +174,9 @@ public void testAddRejected() throws Exception {
             lh.asyncAddEntry(content, new AddCallback() {
                 @Override
                 public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
+                    if (rc != Code.OK) {
+                        receivedException.set(true);
+                    }
                     if (rc == BKException.Code.NotEnoughBookiesException) {
                         receivedTooManyRequestsException.set(true);
                     }
@@ -126,15 +185,19 @@ public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
                 }
             }, null);
         }
-
         counter.await();
 
+        assertTrue(receivedException.get());
         assertTrue(receivedTooManyRequestsException.get());
     }
 
     @Test
     public void testRecoveryNotRejected() throws Exception {
-        LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
+        ServerConfiguration conf = newServerConfiguration();
+        bsConfs.add(conf);
+        bs.add(startBookie(conf));
+
+        @Cleanup LedgerHandle lh = bkc.createLedger(1, 1, digestType, new byte[0]);
         byte[] content = new byte[100];
 
         final int numEntriesToRead = 1000;
@@ -146,10 +209,11 @@ public void testRecoveryNotRejected() throws Exception {
         final int numLedgersToRecover = 10;
         List<Long> ledgersToRecover = Lists.newArrayList();
         for (int i = 0; i < numLedgersToRecover; i++) {
-            LedgerHandle lhr = bkc.createLedger(1, 1, digestType, new byte[0]);
-            lhr.addEntry(content);
-            // Leave the ledger in open state
-            ledgersToRecover.add(lhr.getId());
+            try (LedgerHandle lhr = bkc.createLedger(1, 1, digestType, new byte[0])) {
+                lhr.addEntry(content);
+                // Leave the ledger in open state
+                ledgersToRecover.add(lhr.getId());
+            }
         }
 
         ExecutorService executor = Executors.newCachedThreadPool();
@@ -171,15 +235,12 @@ public Void call() throws Exception {
         }));
 
         for (long ledgerId : ledgersToRecover) {
-            futures.add(executor.submit(new Callable<Void>() {
-                @Override
-                public Void call() throws Exception {
-                    barrier.await();
+            futures.add(executor.submit((Callable<Void>) () -> {
+                barrier.await();
 
-                    // Recovery should always succeed
-                    bkc.openLedger(ledgerId, digestType, new byte[0]);
-                    return null;
-                }
+                // Recovery should always succeed
+                try (LedgerHandle ignored = bkc.openLedger(ledgerId, digestType, new byte[0])) {}
+                return null;
             }));
         }
 
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
index aa01e65da..5fe5fa232 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
@@ -56,6 +56,10 @@
     public UpdateLedgerOpTest() {
         super(3);
         baseConf.setGcWaitTime(100000);
+        baseConf.setMaxPendingAddRequestPerThread(0);
+        baseConf.setMaxPendingReadRequestPerThread(0);
+        baseClientConf.setAddEntryQuorumTimeout(Integer.MAX_VALUE);
+        baseClientConf.setAddEntryTimeout(Integer.MAX_VALUE);
     }
 
     UpdateLedgerNotifier progressable = new UpdateLedgerNotifier() {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
index aba12a748..09681f3a6 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
@@ -53,6 +53,9 @@ public static ServerConfiguration newServerConfiguration() {
         confReturn.setGcWaitTime(1000);
         confReturn.setDiskUsageThreshold(0.999f);
         confReturn.setDiskUsageWarnThreshold(0.99f);
+        // disable queue limit
+        confReturn.setMaxPendingReadRequestPerThread(0);
+        confReturn.setMaxPendingAddRequestPerThread(0);
         setLoopbackInterfaceAndAllowLoopback(confReturn);
         return confReturn;
     }
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
index 0147c06e5..909e38b5d 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
@@ -37,6 +37,7 @@
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.TimeUnit;
 
+import java.util.function.Supplier;
 import org.apache.bookkeeper.bookie.Bookie;
 import org.apache.bookkeeper.bookie.BookieException;
 import org.apache.bookkeeper.client.BookKeeperTestClient;
@@ -94,7 +95,10 @@
      * bind to loopback address.
      */
     protected final ServerConfiguration baseConf = TestBKConfiguration.newServerConfiguration();
-    protected final ClientConfiguration baseClientConf = new ClientConfiguration();
+    protected final ClientConfiguration baseClientConf = new ClientConfiguration()
+        .setAddEntryQuorumTimeout(Integer.MAX_VALUE)
+        .setAddEntryTimeout(Integer.MAX_VALUE)
+        .setReadEntryTimeout(Integer.MAX_VALUE);
 
     private final Map<BookieServer, AutoRecoveryMain> autoRecoveryProcesses = new HashMap<>();
 
@@ -642,11 +646,20 @@ protected BookieServer startBookie(ServerConfiguration conf)
      */
     protected BookieServer startBookie(ServerConfiguration conf, final Bookie b)
             throws Exception {
+        return startBookie(conf, () -> b);
+    }
+    /**
+     * Start a bookie with the given bookie instance. Also, starts the auto
+     * recovery for this bookie, if isAutoRecoveryEnabled is true.
+     */
+    protected BookieServer startBookie(ServerConfiguration conf,
+                                       final Supplier<Bookie> bookieSupplier)
+            throws Exception {
         TestStatsProvider provider = new TestStatsProvider();
         BookieServer server = new BookieServer(conf, provider.getStatsLogger("")) {
             @Override
             protected Bookie newBookie(ServerConfiguration conf) {
-                return b;
+                return bookieSupplier.get();
             }
         };
 
@@ -661,6 +674,7 @@ protected Bookie newBookie(ServerConfiguration conf) {
         server.start();
         bsLoggers.put(server.getLocalAddress(), provider);
 
+        LOG.info("Waiting bookie '{}' to be created.", address);
         waitForBookie.get(30, TimeUnit.SECONDS);
         LOG.info("New bookie '{}' has been created.", address);
 
diff --git a/pom.xml b/pom.xml
index b69605906..814e3190e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,6 +103,8 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <javac.target>1.8</javac.target>
+    <!-- plugin settings -->
+    <bk.leakDetectionLevel>PARANOID</bk.leakDetectionLevel>
     <!-- dependencies -->
     <arquillian-cube.version>1.15.1</arquillian-cube.version>
     <arquillian-junit.version>1.1.14.Final</arquillian-junit.version>
@@ -723,7 +725,7 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${maven-surefire-plugin.version}</version>
         <configuration>
-          <argLine>-Xmx2G -Djava.net.preferIPv4Stack=true</argLine>
+          <argLine>-Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetectionLevel=${bk.leakDetectionLevel}</argLine>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
           <reuseForks>false</reuseForks>
           <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
@@ -852,7 +854,7 @@
             <version>${maven-surefire-plugin.version}</version>
             <configuration>
               <!-- @{argLine} is a variable injected by JaCoCo-->
-              <argLine>@{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true</argLine>
+              <argLine>@{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=${bk.leakDetectionLevel}</argLine>
               <redirectTestOutputToFile>true</redirectTestOutputToFile>
               <reuseForks>false</reuseForks>
               <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml
index c99b5513a..a6a2d3e5b 100644
--- a/stream/distributedlog/common/pom.xml
+++ b/stream/distributedlog/common/pom.xml
@@ -99,7 +99,7 @@
         <version>${maven-surefire-plugin.version}</version>
         <configuration>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
-          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G</argLine>
+          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID</argLine>
           <forkMode>always</forkMode>
           <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
         </configuration>
diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml
index 547ad01a6..fc910f2a4 100644
--- a/stream/distributedlog/core/pom.xml
+++ b/stream/distributedlog/core/pom.xml
@@ -91,7 +91,7 @@
         <version>${maven-surefire-plugin.version}</version>
         <configuration>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
-          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G</argLine>
+          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID</argLine>
           <forkMode>always</forkMode>
           <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
           <properties>
diff --git a/stream/tests/integration/pom.xml b/stream/tests/integration/pom.xml
index 75acec183..ed8850d1a 100644
--- a/stream/tests/integration/pom.xml
+++ b/stream/tests/integration/pom.xml
@@ -68,7 +68,7 @@
           <!-- only run tests when -DstreamIntegrationTests is specified //-->
           <skipTests>true</skipTests>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
-          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G</argLine>
+          <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetectionLevel=${bk.leakDetectionLevel}</argLine>
           <forkMode>always</forkMode>
           <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
         </configuration>


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services