You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by fp...@apache.org on 2011/08/11 21:37:21 UTC
svn commit: r1156766 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/
bookkeeper-server/src/main/java/org/apache/bookkeeper/client/
bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ bookkeep...
Author: fpj
Date: Thu Aug 11 19:37:21 2011
New Revision: 1156766
URL: http://svn.apache.org/viewvc?rev=1156766&view=rev
Log:
BOOKKEEPER-29: BookieRecoveryTest fails intermittently (ivank, fpj via fpj)
Added:
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieFailureTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieReadWriteTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieRecoveryTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Thu Aug 11 19:37:21 2011
@@ -28,3 +28,5 @@ BUGFIXES:
BOOKKEEPER-33: Add length and offset parameter to addEntry (ivank via fpj)
+ BOOKKEEPER-29: BookieRecoveryTest fails intermittently (ivank, fpj via fpj)
+
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java Thu Aug 11 19:37:21 2011
@@ -135,7 +135,6 @@ public class Bookie extends Thread {
}
SyncThread syncThread = new SyncThread();
public Bookie(int port, String zkServers, File journalDirectory, File ledgerDirectories[]) throws IOException {
- instantiateZookeeperClient(port, zkServers);
this.journalDirectory = journalDirectory;
this.ledgerDirectories = ledgerDirectories;
entryLogger = new EntryLogger(ledgerDirectories, this);
@@ -202,6 +201,7 @@ public class Bookie extends Thread {
}
}
}
+ instantiateZookeeperClient(port, zkServers);
setDaemon(true);
LOG.debug("I'm starting a bookie with journal directory " + journalDirectory.getName());
start();
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java Thu Aug 11 19:37:21 2011
@@ -79,7 +79,7 @@ class BookieWatcher implements Watcher,
}
public void readBookies(ChildrenCallback callback) {
- bk.getZkHandle().getChildren( BOOKIE_REGISTRATION_PATH, this, callback, null);
+ bk.getZkHandle().getChildren(BOOKIE_REGISTRATION_PATH, this, callback, null);
}
@Override
Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java?rev=1156766&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java Thu Aug 11 19:37:21 2011
@@ -0,0 +1,61 @@
+package org.apache.bookkeeper.client;
+
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+
+import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
+import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback;
+import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
+import org.apache.bookkeeper.client.BKException.Code;
+import org.apache.bookkeeper.proto.BookieClient;
+import org.apache.bookkeeper.util.OrderedSafeExecutor;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+
+/**
+ * Test BookKeeperClient which allows access to members we don't
+ * wish to expose in the public API.
+ */
+public class BookKeeperTestClient extends BookKeeper {
+ public BookKeeperTestClient(String servers)
+ throws IOException, InterruptedException, KeeperException {
+ super(servers);
+ }
+
+ /**
+ * Force a read to zookeeper to get list of bookies.
+ *
+ * @throws InterruptedException
+ * @throws KeeperException
+ */
+ public void readBookiesBlocking()
+ throws InterruptedException, KeeperException {
+ bookieWatcher.readBookiesBlocking();
+ }
+}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java Thu Aug 11 19:37:21 2011
@@ -28,7 +28,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperTestClient;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.log4j.Logger;
@@ -64,7 +64,7 @@ public abstract class BaseTestCase exten
List<BookieServer> bs = new ArrayList<BookieServer>();
Integer initialPort = 5000;
int numBookies;
- BookKeeper bkc;
+ BookKeeperTestClient bkc;
public BaseTestCase(int numBookies) {
this.numBookies = numBookies;
@@ -117,7 +117,7 @@ public abstract class BaseTestCase exten
bs.add(server);
}
zkc.close();
- bkc = new BookKeeper("127.0.0.1");
+ bkc = new BookKeeperTestClient("127.0.0.1");
} catch(Exception e) {
LOG.error("Error setting up", e);
throw e;
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieFailureTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieFailureTest.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieFailureTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieFailureTest.java Thu Aug 11 19:37:21 2011
@@ -31,7 +31,7 @@ import java.util.Set;
import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
import org.apache.bookkeeper.client.BKException;
-import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperTestClient;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.AsyncCallback.ReadCallback;
@@ -121,8 +121,6 @@ public class BookieFailureTest extends B
@Test
public void testBookieRecovery() throws Exception{
- bkc = new BookKeeper("127.0.0.1");
-
//Shutdown all but 1 bookie
bs.get(0).shutdown();
bs.get(1).shutdown();
@@ -157,7 +155,7 @@ public class BookieFailureTest extends B
void auxTestReadWriteAsyncSingleClient(BookieServer bs) throws IOException {
try {
- // Create a BookKeeper client and a ledger
+ // Create a ledger
lh = bkc.createLedger(3, 2, digestType, ledgerPassword);
ledgerId = lh.getId();
@@ -193,7 +191,7 @@ public class BookieFailureTest extends B
// open ledger
bkc.halt();
- bkc = new BookKeeper("127.0.0.1");
+ bkc = new BookKeeperTestClient("127.0.0.1");
lh = bkc.openLedger(ledgerId, digestType, ledgerPassword);
LOG.debug("Number of entries written: " + (lh.getLastAddConfirmed() + 1));
assertTrue("Verifying number of entries written", lh.getLastAddConfirmed() == (numEntriesToWrite - 1));
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieReadWriteTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieReadWriteTest.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieReadWriteTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieReadWriteTest.java Thu Aug 11 19:37:21 2011
@@ -119,7 +119,6 @@ implements AddCallback, ReadCallback, Re
*/
@Test
public void testStreamingClients() throws IOException, KeeperException, BKException, InterruptedException {
- bkc = new BookKeeper("127.0.0.1");
lh = bkc.createLedger(digestType, ledgerPassword);
// write a string so that we cna
// create a buffer of a single bytes
@@ -169,8 +168,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testReadWriteAsyncSingleClient() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -255,8 +253,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testReadWriteRangeAsyncSingleClient() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -423,9 +420,8 @@ implements AddCallback, ReadCallback, Re
Integer throttle = 100;
ThrottleTestCallback tcb = new ThrottleTestCallback(throttle);
- // Create a BookKeeper client and a ledger
+ // Create a ledger
System.setProperty("throttle", throttle.toString());
- bkc = new BookKeeper("127.0.0.1");
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -519,8 +515,7 @@ implements AddCallback, ReadCallback, Re
String charset = "utf-8";
LOG.debug("Default charset: " + Charset.defaultCharset());
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -593,8 +588,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testReadWriteSyncSingleClient() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -640,8 +634,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testReadWriteZero() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -688,8 +681,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testMultiLedger() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
lh2 = bkc.createLedger(digestType, ledgerPassword);
@@ -748,8 +740,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testReadWriteAsyncLength() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -804,7 +795,6 @@ implements AddCallback, ReadCallback, Re
int numLedgers = 10000;
Long throttle = (((Double) Math.max(1.0, ((double) 10000/numLedgers))).longValue());
System.setProperty("throttle", throttle.toString());
- bkc = new BookKeeper("127.0.0.1");
LedgerHandle[] lhArray = new LedgerHandle[numLedgers];
for(int i = 0; i < numLedgers; i++){
lhArray[i] = bkc.createLedger(3, 2, BookKeeper.DigestType.CRC32, new byte[] {'a', 'b'});
@@ -852,8 +842,7 @@ implements AddCallback, ReadCallback, Re
public void testReadFromOpenLedger() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
@@ -931,8 +920,7 @@ implements AddCallback, ReadCallback, Re
@Test
public void testLastConfirmedAdd() throws IOException {
try {
- // Create a BookKeeper client and a ledger
- bkc = new BookKeeper("127.0.0.1");
+ // Create a ledger
lh = bkc.createLedger(digestType, ledgerPassword);
// bkc.initMessageDigest("SHA1");
ledgerId = lh.getId();
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieRecoveryTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieRecoveryTest.java?rev=1156766&r1=1156765&r2=1156766&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieRecoveryTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieRecoveryTest.java Thu Aug 11 19:37:21 2011
@@ -48,7 +48,7 @@ import org.junit.Test;
/**
* This class tests the bookie recovery admin functionality.
*/
-public class BookieRecoveryTest extends BaseTestCase implements Watcher {
+public class BookieRecoveryTest extends BaseTestCase {
static Logger LOG = Logger.getLogger(BookieRecoveryTest.class);
// Object used for synchronizing async method calls
@@ -77,7 +77,7 @@ public class BookieRecoveryTest extends
// Objects to use for this jUnit test.
DigestType digestType;
- SyncObject sync, zkSync;
+ SyncObject sync;
BookieRecoverCallback bookieRecoverCb;
BookKeeperTools bkTools;
@@ -95,7 +95,6 @@ public class BookieRecoveryTest extends
System.setProperty("digestType", digestType.toString());
System.setProperty("passwd", "");
sync = new SyncObject();
- zkSync = new SyncObject();
bookieRecoverCb = new BookieRecoverCallback();
bkTools = new BookKeeperTools(HOSTPORT);
}
@@ -163,24 +162,18 @@ public class BookieRecoveryTest extends
tmpDirs.add(f);
f.delete();
f.mkdir();
- zkSync.value = false;
- bkc.getZkHandle().getChildren("/ledgers/available", this);
BookieServer server = new BookieServer(port, HOSTPORT, f, new File[] { f });
server.start();
bs.add(server);
- while(!zkSync.value){
+ while(bkc.getZkHandle().exists("/ledgers/available/" + InetAddress.getLocalHost().getHostAddress() + ":" + port, false) == null){
Thread.sleep(500);
}
+ bkc.readBookiesBlocking();
LOG.info("New bookie on port " + port + " has been created.");
}
-
- @Override
- public void process(WatchedEvent event) {
- zkSync.value = true;
- }
/**
* Helper method to verify that we can read the recovered ledger entries.
@@ -234,16 +227,9 @@ public class BookieRecoveryTest extends
writeEntriestoLedgers(numMsgs, 0, lhs);
// Shutdown the first bookie server
- zkSync.value = false;
- bkc.getZkHandle().getChildren("/ledgers/available", this);
LOG.info("Finished writing all ledger entries so shutdown one of the bookies.");
bs.get(0).shutdown();
bs.remove(0);
-
- // Block until I get a notification
- while(!zkSync.value){
- Thread.sleep(100);
- }
// Startup a new bookie server
int newBookiePort = initialPort + numBookies;
@@ -294,17 +280,10 @@ public class BookieRecoveryTest extends
writeEntriestoLedgers(numMsgs, 0, lhs);
// Shutdown the first bookie server
- zkSync.value = false;
- bkc.getZkHandle().getChildren("/ledgers/available", this);
LOG.info("Finished writing all ledger entries so shutdown one of the bookies.");
bs.get(0).shutdown();
bs.remove(0);
- // Block until I get a notification
- while(!zkSync.value){
- Thread.sleep(100);
- }
-
// Startup three new bookie servers
for (int i = 0; i < 3; i++) {
int newBookiePort = initialPort + numBookies + i;
@@ -355,16 +334,9 @@ public class BookieRecoveryTest extends
writeEntriestoLedgers(numMsgs, 0, lhs);
// Shutdown the first bookie server
- zkSync.value = false;
- bkc.getZkHandle().getChildren("/ledgers/available", this);
LOG.info("Finished writing all ledger entries so shutdown one of the bookies.");
bs.get(0).shutdown();
bs.remove(0);
-
- // Block until I get a notification
- while(!zkSync.value){
- Thread.sleep(100);
- }
// Startup a new bookie server
int newBookiePort = initialPort + numBookies;
@@ -406,16 +378,10 @@ public class BookieRecoveryTest extends
writeEntriestoLedgers(numMsgs, 0, lhs);
// Shutdown the first bookie server
- zkSync.value = false;
- bkc.getZkHandle().getChildren("/ledgers/available", this);
LOG.info("Finished writing all ledger entries so shutdown one of the bookies.");
bs.get(0).shutdown();
bs.remove(0);
- // Block until I get a notification
- while(!zkSync.value){
- Thread.sleep(100);
- }
// Startup three new bookie servers
for (int i = 0; i < 3; i++) {
int newBookiePort = initialPort + numBookies + i;