You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by fp...@apache.org on 2014/09/11 00:41:28 UTC
svn commit: r1624161 - in /zookeeper/bookkeeper/trunk: CHANGES.txt
bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java
Author: fpj
Date: Wed Sep 10 22:41:28 2014
New Revision: 1624161
URL: http://svn.apache.org/r1624161
Log:
BOOKKEEPER-775: Improve MultipleThreadReadTest to reduce flakiness (sijie via fpj)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1624161&r1=1624160&r2=1624161&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Sep 10 22:41:28 2014
@@ -226,6 +226,8 @@ Trunk (unreleased changes)
BOOKKEEPER-736: Stats for AutoRecovery (sijie)
+ BOOKKEEPER-775: Improve MultipleThreadReadTest to reduce flakiness (sijie via fpj)
+
bookkeeper-benchmark:
BOOKKEEPER-768: fix typo 'seconds' to milliseconds in benchmark output (jialin via sijie)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java?rev=1624161&r1=1624160&r2=1624161&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultipleThreadReadTest.java Wed Sep 10 22:41:28 2014
@@ -1,5 +1,3 @@
-package org.apache.bookkeeper.test;
-
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -20,6 +18,7 @@ package org.apache.bookkeeper.test;
* under the License.
*
*/
+package org.apache.bookkeeper.test;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,11 +26,10 @@ import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.NoSuchElementException;
-
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.bookkeeper.client.AsyncCallback;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.BookKeeperTestClient;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
@@ -41,22 +39,21 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.*;
+import static com.google.common.base.Charsets.UTF_8;
public class MultipleThreadReadTest extends BookKeeperClusterTestCase {
static Logger LOG = LoggerFactory.getLogger(MultipleThreadReadTest.class);
+
BookKeeper.DigestType digestType;
- byte[] ledgerPassword = "aaa".getBytes();
+ byte [] ledgerPassword = "aaa".getBytes();
private int entriesPerLedger = 1000;
final SyncObj mainSyncObj = new SyncObj();
class SyncObj {
volatile int counter;
- boolean value;
boolean failed;
-
public SyncObj() {
counter = 0;
- value = false;
failed = false;
}
}
@@ -64,8 +61,9 @@ public class MultipleThreadReadTest exte
final List<BookKeeperTestClient> clients = new ArrayList<BookKeeperTestClient>();
public MultipleThreadReadTest() {
- super(3);
- this.digestType = DigestType.CRC32;
+ super(6);
+ this.digestType = BookKeeper.DigestType.CRC32;
+ baseClientConf.setAddEntryTimeout(20);
}
private void createClients(int numClients) {
@@ -94,29 +92,25 @@ public class MultipleThreadReadTest exte
clients.clear();
}
- private Thread getWriterThread(final int tNo, final LedgerHandle lh) {
+ private Thread getWriterThread(final int tNo, final LedgerHandle lh, final AtomicBoolean resultHolder) {
Thread t = new Thread(new Runnable() {
@Override
public void run() {
final SyncObj tSync = new SyncObj();
for (int j = 0; j < entriesPerLedger; j++) {
- byte[] entry = ("Entry-" + tNo + "-" + j).getBytes();
+ final byte[] entry = ("Entry-" + tNo + "-" + j).getBytes();
lh.asyncAddEntry(entry, new AsyncCallback.AddCallback() {
@Override
public void addComplete(int rc, LedgerHandle ledgerHandle, long eid, Object o) {
- SyncObj syncObj = (SyncObj) o;
- try {
+ SyncObj syncObj = (SyncObj)o;
+ synchronized (syncObj) {
if (rc != BKException.Code.OK) {
- fail("Add entries returned a code that is not OK. rc:" + rc);
- }
- synchronized (syncObj) {
- syncObj.counter++;
- syncObj.notify();
- }
- } catch (AssertionError e) {
- synchronized (syncObj) {
+ LOG.error("Add entry {} failed : rc = {}", new String(entry, UTF_8), rc);
syncObj.failed = true;
syncObj.notify();
+ } else {
+ syncObj.counter++;
+ syncObj.notify();
}
}
}
@@ -130,20 +124,19 @@ public class MultipleThreadReadTest exte
Thread.currentThread().interrupt();
}
}
- if (tSync.failed) {
- fail("Failed to add entries.");
- }
+ resultHolder.set(!tSync.failed);
}
// close this handle
try {
lh.close();
- } catch (Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
+ } catch (InterruptedException ie) {
+ LOG.error("Interrupted on closing ledger handle {} : ", lh.getId(), ie);
+ Thread.currentThread().interrupt();
+ } catch (BKException bke) {
+ LOG.error("Error on closing ledger handle {} : ", lh.getId(), bke);
}
}
- });
+ }, "WriteThread(Lid=" + lh.getId() + ")");
t.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread thread, Throwable throwable) {
@@ -155,66 +148,73 @@ public class MultipleThreadReadTest exte
return t;
}
- private Thread getReaderThread(final int tNo, final LedgerHandle lh, final int ledgerNumber) {
+ private Thread getReaderThread(final int tNo, final LedgerHandle lh, final int ledgerNumber,
+ final AtomicBoolean resultHolder) {
Thread t = new Thread(new Runnable() {
@Override
public void run() {
- try {
- final SyncObj tSync = new SyncObj();
- lh.asyncReadEntries(0, entriesPerLedger - 1, new AsyncCallback.ReadCallback() {
- @Override
- public void readComplete(int rc, LedgerHandle ledgerHandle, Enumeration<LedgerEntry> list,
- Object o) {
- SyncObj syncObj = (SyncObj) o;
+ //LedgerHandle lh = clientList.get(0).openLedger(ledgerIds.get(tNo % numLedgers), digestType, ledgerPassword);
+ long startEntryId = 0;
+ long endEntryId;
+ long eid = 0;
+ while (startEntryId <= entriesPerLedger - 1) {
+ endEntryId = Math.min(startEntryId + 50 - 1, entriesPerLedger - 1);
+ final long numEntries = (endEntryId - startEntryId) + 1;
+ boolean success = true;
+ try {
+ Enumeration<LedgerEntry> list = lh.readEntries(startEntryId, endEntryId);
+ for (int j = 0; j < numEntries; j++) {
+ LedgerEntry e;
try {
- if (rc != BKException.Code.OK) {
- fail("Read entries returned a code that is not OK. rc:" + rc);
- }
- for (int j = 0; j < entriesPerLedger; j++) {
- LedgerEntry e = null;
- try {
- e = list.nextElement();
- } catch (NoSuchElementException exception) {
- fail("Short read for ledger:" + ledgerHandle.getId());
- }
- byte[] data = e.getEntry();
- if (!Arrays.equals(("Entry-" + ledgerNumber + "-" + j).getBytes(), data)) {
- fail("Wrong entry while reading from ledger");
- }
- }
- if (list.hasMoreElements()) {
- fail("Read more elements than we wrote for ledger:" + ledgerHandle.getId());
- }
- } catch (AssertionError e) {
- synchronized (syncObj) {
- syncObj.failed = true;
- syncObj.notify();
- }
- } finally {
- synchronized (syncObj) {
- syncObj.value = true;
- syncObj.notify();
- }
+ e = list.nextElement();
+ } catch (NoSuchElementException exception) {
+ success = false;
+ break;
+ }
+ long curEid = eid++;
+ if (e.getEntryId() != curEid) {
+ LOG.error("Expected entry id {} for ledger {} but {} found.",
+ new Object[] { curEid, lh.getId(), e.getEntryId() });
+ success = false;
+ break;
+ }
+ byte[] data = e.getEntry();
+ if (!Arrays.equals(("Entry-" + ledgerNumber + "-" + e.getEntryId()).getBytes(), data)) {
+ LOG.error("Expected entry data 'Entry-{}-{}' but {} found for ledger {}.",
+ new Object[] { ledgerNumber, e.getEntryId(), new String(data, UTF_8), lh.getId() });
+ success = false;
+ break;
}
}
- }, tSync);
- synchronized (tSync) {
- while (!tSync.value) {
- tSync.wait();
- }
- if (tSync.failed) {
- fail("Invalid read while reading form ledger:" + lh.getId());
+ if (success) {
+ success = !list.hasMoreElements();
+ if (!success) {
+ LOG.error("Found more entries returned on reading ({}-{}) from ledger {}.",
+ new Object[] { startEntryId, endEntryId, lh.getId() });
+ }
}
+ } catch (InterruptedException ie) {
+ LOG.error("Interrupted on reading entries ({} - {}) from ledger {} : ",
+ new Object[] { startEntryId, endEntryId, lh.getId(), ie });
+ Thread.currentThread().interrupt();
+ success = false;
+ } catch (BKException bke) {
+ LOG.error("Failed on reading entries ({} - {}) from ledger {} : ",
+ new Object[] { startEntryId, endEntryId, lh.getId(), bke });
+ success = false;
}
- } catch (InterruptedException e) {
- fail("Interrupted while waiting for replies.");
- Thread.currentThread().interrupt();
+ resultHolder.set(success);
+ if (!success) {
+ break;
+ }
+ startEntryId = endEntryId + 1;
}
}
- });
+ }, "ReadThread(Tid =" + tNo + ", Lid=" + lh.getId() + ")");
t.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread thread, Throwable throwable) {
+ LOG.error("Uncaught exception in thread {} : ", thread.getName(), throwable);
synchronized (mainSyncObj) {
mainSyncObj.failed = true;
}
@@ -229,7 +229,7 @@ public class MultipleThreadReadTest exte
* @throws java.io.IOException
*/
public void multiLedgerMultiThreadRead(final int numLedgers,
- final int numThreads) throws IOException {
+ final int numThreads) throws IOException {
assertTrue(numLedgers != 0 && numThreads >= numLedgers && numThreads % numLedgers == 0);
// We create numThread/numLedger clients so that each client can be used to open a handle.
@@ -237,14 +237,17 @@ public class MultipleThreadReadTest exte
final List<LedgerHandle> oldLedgerHandles = new ArrayList<LedgerHandle>();
final List<Long> ledgerIds = new ArrayList<Long>();
List<Thread> threadList = new ArrayList<Thread>();
+ List<AtomicBoolean> writeResults = new ArrayList<AtomicBoolean>();
// Start write threads.
// Only one thread writes to a ledger, so just use numLedgers instead.
for (int i = 0; i < numLedgers; i++) {
LedgerHandle lh = bkc.createLedger(digestType, ledgerPassword);
oldLedgerHandles.add(lh);
ledgerIds.add(lh.getId());
+ AtomicBoolean writeResult = new AtomicBoolean(false);
+ writeResults.add(writeResult);
Thread t;
- threadList.add(t = getWriterThread(i, oldLedgerHandles.get(i)));
+ threadList.add(t = getWriterThread(i, oldLedgerHandles.get(i), writeResult));
t.start();
}
// Wait for the threads to complete
@@ -253,9 +256,13 @@ public class MultipleThreadReadTest exte
}
synchronized (mainSyncObj) {
if (mainSyncObj.failed) {
- fail("Test failed because we couldn't add entries.");
+ fail("Test failed because we encountered uncaught exception on adding entries.");
}
}
+ for (int i = 0; i < numLedgers; i++) {
+ assertTrue("Failed on adding entries for ledger " + oldLedgerHandles.get(i).getId(),
+ writeResults.get(i).get());
+ }
// Close the ledger handles.
for (LedgerHandle lh : oldLedgerHandles) {
try {
@@ -274,10 +281,13 @@ public class MultipleThreadReadTest exte
closeClientsAndClear();
createClients(numThreads / numLedgers);
+ List<AtomicBoolean> readResults = new ArrayList<AtomicBoolean>();
for (int i = 0; i < numThreads; i++) {
+ AtomicBoolean readResult = new AtomicBoolean(false);
Thread t;
threadList.add(t = getReaderThread(i, clients.get(i / numLedgers)
- .openLedger(ledgerIds.get(i % numLedgers), digestType, ledgerPassword), i % numLedgers));
+ .openLedger(ledgerIds.get(i % numLedgers), digestType, ledgerPassword), i % numLedgers, readResult));
+ readResults.add(readResult);
t.start();
}
// Wait for the threads to complete.
@@ -286,9 +296,12 @@ public class MultipleThreadReadTest exte
}
synchronized (mainSyncObj) {
if (mainSyncObj.failed) {
- fail("Test failed because we couldn't read entries");
+ fail("Test failed because we encountered uncaught exception on reading entries");
}
}
+ for (AtomicBoolean readResult : readResults) {
+ assertTrue("Failed on read entries", readResult.get());
+ }
} catch (BKException e) {
LOG.error("Test failed", e);
fail("Test failed due to BookKeeper exception");
@@ -298,17 +311,17 @@ public class MultipleThreadReadTest exte
}
}
- @Test
+ @Test(timeout = 60000)
public void test10Ledgers20ThreadsRead() throws IOException {
multiLedgerMultiThreadRead(10, 20);
}
- @Test
+ @Test(timeout = 60000)
public void test10Ledgers200ThreadsRead() throws IOException {
multiLedgerMultiThreadRead(10, 200);
}
- @Test
+ @Test(timeout = 60000)
public void test1Ledger50ThreadsRead() throws IOException {
multiLedgerMultiThreadRead(1, 50);
}