You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by ph...@apache.org on 2012/03/15 17:50:17 UTC
svn commit: r1301082 - in /zookeeper/branches/branch-3.4: ./
src/java/main/org/apache/zookeeper/server/
src/java/main/org/apache/zookeeper/server/quorum/
src/java/test/org/apache/zookeeper/server/
src/java/test/org/apache/zookeeper/test/
Author: phunt
Date: Thu Mar 15 16:50:16 2012
New Revision: 1301082
URL: http://svn.apache.org/viewvc?rev=1301082&view=rev
Log:
ZOOKEEPER-1277. servers stop serving when lower 32bits of zxid roll over (phunt)
Added:
zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZxidRolloverTest.java
Modified:
zookeeper/branches/branch-3.4/CHANGES.txt
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/RequestProcessor.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ProposalRequestProcessor.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyRequestProcessor.java
zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/ClientBase.java
Modified: zookeeper/branches/branch-3.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/CHANGES.txt?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/CHANGES.txt (original)
+++ zookeeper/branches/branch-3.4/CHANGES.txt Thu Mar 15 16:50:16 2012
@@ -8,8 +8,11 @@ BUGFIXES:
replace "http://java.sun.com/javase/6/docs/api/" with
"http://download.oracle.com/javase/6/docs/api/" (Eugene Koontz via camille)
- ZOOKEEPER-1354. AuthTest.testBadAuthThenSendOtherCommands fails intermittently (phunt via camille)
+ ZOOKEEPER-1354. AuthTest.testBadAuthThenSendOtherCommands fails
+ intermittently (phunt via camille)
+ ZOOKEEPER-1277. servers stop serving when lower 32bits of zxid roll
+ over (phunt)
IMPROVEMENTS:
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java Thu Mar 15 16:50:16 2012
@@ -56,6 +56,7 @@ import org.apache.zookeeper.proto.CheckV
import org.apache.zookeeper.server.ZooKeeperServer.ChangeRecord;
import org.apache.zookeeper.server.auth.AuthenticationProvider;
import org.apache.zookeeper.server.auth.ProviderRegistry;
+import org.apache.zookeeper.server.quorum.Leader.XidRolloverException;
import org.apache.zookeeper.txn.CreateSessionTxn;
import org.apache.zookeeper.txn.CreateTxn;
import org.apache.zookeeper.txn.DeleteTxn;
@@ -131,6 +132,13 @@ public class PrepRequestProcessor extend
}
} catch (InterruptedException e) {
LOG.error("Unexpected interruption", e);
+ } catch (RequestProcessorException e) {
+ if (e.getCause() instanceof XidRolloverException) {
+ LOG.info(e.getCause().getMessage());
+ }
+ LOG.error("Unexpected exception", e);
+ } catch (Exception e) {
+ LOG.error("Unexpected exception", e);
}
LOG.info("PrepRequestProcessor exited loop!");
}
@@ -292,7 +300,9 @@ public class PrepRequestProcessor extend
* @param record
*/
@SuppressWarnings("unchecked")
- protected void pRequest2Txn(int type, long zxid, Request request, Record record, boolean deserialize) throws KeeperException, IOException {
+ protected void pRequest2Txn(int type, long zxid, Request request, Record record, boolean deserialize)
+ throws KeeperException, IOException, RequestProcessorException
+ {
request.hdr = new TxnHeader(request.sessionId, request.cxid, zxid,
zks.getTime(), type);
@@ -493,7 +503,7 @@ public class PrepRequestProcessor extend
* @param request
*/
@SuppressWarnings("unchecked")
- protected void pRequest(Request request) {
+ protected void pRequest(Request request) throws RequestProcessorException {
// LOG.info("Prep>>> cxid = " + request.cxid + " type = " +
// request.type + " id = 0x" + Long.toHexString(request.sessionId));
request.hdr = null;
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/RequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/RequestProcessor.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/RequestProcessor.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/RequestProcessor.java Thu Mar 15 16:50:16 2012
@@ -31,7 +31,14 @@ package org.apache.zookeeper.server;
* any RequestProcessors that it is connected to.
*/
public interface RequestProcessor {
- void processRequest(Request request);
+ @SuppressWarnings("serial")
+ public static class RequestProcessorException extends Exception {
+ public RequestProcessorException(String msg, Throwable t) {
+ super(msg, t);
+ }
+ }
+
+ void processRequest(Request request) throws RequestProcessorException;
void shutdown();
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java Thu Mar 15 16:50:16 2012
@@ -155,7 +155,9 @@ public class SyncRequestProcessor extend
LOG.info("SyncRequestProcessor exited!");
}
- private void flush(LinkedList<Request> toFlush) throws IOException {
+ private void flush(LinkedList<Request> toFlush)
+ throws IOException, RequestProcessorException
+ {
if (toFlush.isEmpty())
return;
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java Thu Mar 15 16:50:16 2012
@@ -18,19 +18,10 @@
package org.apache.zookeeper.server;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -39,13 +30,12 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
+
+import javax.security.sasl.SaslException;
import org.apache.jute.BinaryInputArchive;
import org.apache.jute.BinaryOutputArchive;
import org.apache.jute.Record;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.zookeeper.Environment;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.Code;
@@ -63,6 +53,7 @@ import org.apache.zookeeper.proto.ReplyH
import org.apache.zookeeper.proto.RequestHeader;
import org.apache.zookeeper.proto.SetSASLResponse;
import org.apache.zookeeper.server.DataTree.ProcessTxnResult;
+import org.apache.zookeeper.server.RequestProcessor.RequestProcessorException;
import org.apache.zookeeper.server.ServerCnxn.CloseRequestException;
import org.apache.zookeeper.server.SessionTracker.Session;
import org.apache.zookeeper.server.SessionTracker.SessionExpirer;
@@ -72,9 +63,9 @@ import org.apache.zookeeper.server.persi
import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer;
import org.apache.zookeeper.txn.CreateSessionTxn;
import org.apache.zookeeper.txn.TxnHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.zookeeper.server.util.ZxidUtils;
-import javax.security.sasl.SaslException;
/**
* This class implements a simple standalone ZooKeeperServer. It sets up the
@@ -666,6 +657,8 @@ public class ZooKeeperServer implements
if (LOG.isDebugEnabled()) {
LOG.debug("Dropping request: " + e.getMessage());
}
+ } catch (RequestProcessorException e) {
+ LOG.error("Unable to process request:" + e.getMessage(), e);
}
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java Thu Mar 15 16:50:16 2012
@@ -362,6 +362,24 @@ public class Leader {
self.tick++;
}
+ /**
+ * WARNING: do not use this for anything other than QA testing
+ * on a real cluster. Specifically to enable verification that quorum
+ * can handle the lower 32bit roll-over issue identified in
+ * ZOOKEEPER-1277. Without this option it would take a very long
+ * time (on order of a month say) to see the 4 billion writes
+ * necessary to cause the roll-over to occur.
+ *
+ * This field allows you to override the zxid of the server. Typically
+ * you'll want to set it to something like 0xfffffff0 and then
+ * start the quorum, run some operations and see the re-election.
+ */
+ String initialZxid = System.getProperty("zookeeper.testingonly.initialZxid");
+ if (initialZxid != null) {
+ long zxid = Long.parseLong(initialZxid);
+ zk.setZxid((zk.getZxid() & 0xffffffff00000000L) | zxid);
+ }
+
if (!System.getProperty("zookeeper.leaderServes", "yes").equals("no")) {
self.cnxnFactory.setZooKeeperServer(zk);
}
@@ -567,7 +585,7 @@ public class Leader {
*
* @see org.apache.zookeeper.server.RequestProcessor#processRequest(org.apache.zookeeper.server.Request)
*/
- public void processRequest(Request request) {
+ public void processRequest(Request request) throws RequestProcessorException {
// request.addRQRec(">tobe");
next.processRequest(request);
Proposal p = toBeApplied.peek();
@@ -651,13 +669,31 @@ public class Leader {
return ZxidUtils.getEpochFromZxid(lastProposed);
}
+ @SuppressWarnings("serial")
+ public static class XidRolloverException extends Exception {
+ public XidRolloverException(String message) {
+ super(message);
+ }
+ }
+
/**
* create a proposal and send it out to all the members
*
* @param request
* @return the proposal that is queued to send to all the members
*/
- public Proposal propose(Request request) {
+ public Proposal propose(Request request) throws XidRolloverException {
+ /**
+ * Address the rollover issue. All lower 32bits set indicate a new leader
+ * election. Force a re-election instead. See ZOOKEEPER-1277
+ */
+ if ((request.zxid & 0xffffffffL) == 0xffffffffL) {
+ String msg =
+ "zxid lower 32 bits have rolled over, forcing re-election, and therefore new epoch start";
+ shutdown(msg);
+ throw new XidRolloverException(msg);
+ }
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
BinaryOutputArchive boa = BinaryOutputArchive.getArchive(baos);
try {
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ProposalRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ProposalRequestProcessor.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ProposalRequestProcessor.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ProposalRequestProcessor.java Thu Mar 15 16:50:16 2012
@@ -18,12 +18,12 @@
package org.apache.zookeeper.server.quorum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.zookeeper.server.Request;
import org.apache.zookeeper.server.RequestProcessor;
import org.apache.zookeeper.server.SyncRequestProcessor;
-import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.quorum.Leader.XidRolloverException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This RequestProcessor simply forwards requests to an AckRequestProcessor and
@@ -54,7 +54,7 @@ public class ProposalRequestProcessor im
syncProcessor.start();
}
- public void processRequest(Request request) {
+ public void processRequest(Request request) throws RequestProcessorException {
// LOG.warn("Ack>>> cxid = " + request.cxid + " type = " +
// request.type + " id = " + request.sessionId);
// request.addRQRec(">prop");
@@ -74,7 +74,11 @@ public class ProposalRequestProcessor im
nextProcessor.processRequest(request);
if (request.hdr != null) {
// We need to sync and get consensus on any transactions
- zks.getLeader().propose(request);
+ try {
+ zks.getLeader().propose(request);
+ } catch (XidRolloverException e) {
+ throw new RequestProcessorException(e.getMessage(), e);
+ }
syncProcessor.processRequest(request);
}
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyRequestProcessor.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyRequestProcessor.java (original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyRequestProcessor.java Thu Mar 15 16:50:16 2012
@@ -28,6 +28,8 @@ import org.apache.zookeeper.server.Reque
import org.apache.zookeeper.server.RequestProcessor;
import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.zookeeper.server.ZooTrace;
+import org.apache.zookeeper.server.RequestProcessor.RequestProcessorException;
+import org.apache.zookeeper.server.quorum.Leader.XidRolloverException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,6 +99,13 @@ public class ReadOnlyRequestProcessor ex
}
} catch (InterruptedException e) {
LOG.error("Unexpected interruption", e);
+ } catch (RequestProcessorException e) {
+ if (e.getCause() instanceof XidRolloverException) {
+ LOG.info(e.getCause().getMessage());
+ }
+ LOG.error("Unexpected exception", e);
+ } catch (Exception e) {
+ LOG.error("Unexpected exception", e);
}
LOG.info("ReadOnlyRequestProcessor exited loop!");
}
Added: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZxidRolloverTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZxidRolloverTest.java?rev=1301082&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZxidRolloverTest.java (added)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZxidRolloverTest.java Thu Mar 15 16:50:16 2012
@@ -0,0 +1,352 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException.ConnectionLossException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.test.ClientBase;
+import org.apache.zookeeper.test.ClientBase.CountdownWatcher;
+import org.apache.zookeeper.test.ClientTest;
+import org.apache.zookeeper.test.QuorumUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify ZOOKEEPER-1277 - ensure that we handle epoch rollover correctly.
+ */
+public class ZxidRolloverTest extends TestCase {
+ private static final Logger LOG = Logger.getLogger(ZxidRolloverTest.class);
+
+ private QuorumUtil qu;
+ private ZooKeeperServer zksLeader;
+ private ZooKeeper[] zkClients = new ZooKeeper[3];
+ private CountdownWatcher[] zkClientWatchers = new CountdownWatcher[3];
+ private int idxLeader;
+ private int idxFollower;
+
+ private ZooKeeper getClient(int idx) {
+ return zkClients[idx-1];
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ LOG.info("STARTING " + getName());
+
+ // set the snap count to something low so that we force log rollover
+ // and verify that is working as part of the epoch rollover.
+ SyncRequestProcessor.setSnapCount(7);
+
+ qu = new QuorumUtil(1);
+ startAll();
+
+ for (int i = 0; i < zkClients.length; i++) {
+ zkClientWatchers[i] = new CountdownWatcher();
+ int followerPort = qu.getPeer(i+1).peer.getClientPort();
+ zkClients[i] = new ZooKeeper(
+ "127.0.0.1:" + followerPort,
+ ClientTest.CONNECTION_TIMEOUT, zkClientWatchers[i]);
+ }
+ waitForClients();
+ }
+
+ private void waitForClients() throws Exception {
+ for (int i = 0; i < zkClients.length; i++) {
+ zkClientWatchers[i].waitForConnected(ClientTest.CONNECTION_TIMEOUT);
+ zkClientWatchers[i].reset();
+ }
+ }
+
+ private void startAll() throws IOException {
+ qu.startAll();
+ checkLeader();
+ }
+ private void start(int idx) throws IOException {
+ qu.start(idx);
+ for (String hp : qu.getConnString().split(",")) {
+ Assert.assertTrue("waiting for server up", ClientBase.waitForServerUp(hp,
+ ClientTest.CONNECTION_TIMEOUT));
+ }
+
+ checkLeader();
+ }
+
+ private void checkLeader() {
+ idxLeader = 1;
+ while(qu.getPeer(idxLeader).peer.leader == null) {
+ idxLeader++;
+ }
+ idxFollower = (idxLeader == 1 ? 2 : 1);
+
+ zksLeader = qu.getPeer(idxLeader).peer.getActiveServer();
+ }
+
+ private void shutdownAll() throws IOException {
+ qu.shutdownAll();
+ }
+
+ private void shutdown(int idx) throws IOException {
+ qu.shutdown(idx);
+ }
+
+ /** Reset the next zxid to be near epoch end */
+ private void adjustEpochNearEnd() {
+ zksLeader.setZxid((zksLeader.getZxid() & 0xffffffff00000000L) | 0xfffffffcL);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ LOG.info("tearDown starting");
+ for (int i = 0; i < zkClients.length; i++) {
+ zkClients[i].close();
+ }
+ qu.shutdownAll();
+ }
+
+ /**
+ * Create the znodes, this may fail if the lower 32 roll over, if so
+ * wait for the clients to be re-connected after the re-election
+ */
+ private int createNodes(ZooKeeper zk, int start, int count) throws Exception {
+ LOG.info("Creating nodes " + start + " thru " + (start + count));
+ int j = 0;
+ try {
+ for (int i = start; i < start + count; i++) {
+ zk.create("/foo" + i, new byte[0], Ids.READ_ACL_UNSAFE,
+ CreateMode.EPHEMERAL);
+ j++;
+ }
+ } catch (ConnectionLossException e) {
+ // this is ok - the leader has dropped leadership
+ waitForClients();
+ }
+ return j;
+ }
+ /**
+ * Verify the expected znodes were created and that the last znode, which
+ * caused the roll-over, did not.
+ */
+ private void checkNodes(ZooKeeper zk, int start, int count) throws Exception {
+ LOG.info("Validating nodes " + start + " thru " + (start + count));
+ for (int i = start; i < start + count; i++) {
+ assertNotNull(zk.exists("/foo" + i, false));
+ LOG.error("Exists zxid:" + Long.toHexString(zk.exists("/foo" + i, false).getCzxid()));
+ }
+ assertNull(zk.exists("/foo" + (start + count), false));
+ }
+
+ /**
+ * Prior to the fix this test would hang for a while, then fail with
+ * connection loss.
+ */
+ @Test
+ public void testSimpleRolloverFollower() throws Exception {
+ adjustEpochNearEnd();
+
+ ZooKeeper zk = getClient((idxLeader == 1 ? 2 : 1));
+ int countCreated = createNodes(zk, 0, 10);
+
+ checkNodes(zk, 0, countCreated);
+ }
+
+ /**
+ * Similar to testSimpleRollover, but ensure the cluster comes back,
+ * has the right data, and is able to serve new requests.
+ */
+ @Test
+ public void testRolloverThenRestart() throws Exception {
+ ZooKeeper zk = getClient(idxFollower);
+
+ int countCreated = createNodes(zk, 0, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdownAll();
+ startAll();
+ zk = getClient(idxLeader);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdownAll();
+ startAll();
+ zk = getClient(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdownAll();
+ startAll();
+ zk = getClient(idxLeader);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ // sanity check
+ assertTrue(countCreated > 0);
+ assertTrue(countCreated < 60);
+ }
+
+ /**
+ * Similar to testRolloverThenRestart, but ensure a follower comes back,
+ * has the right data, and is able to serve new requests.
+ */
+ @Test
+ public void testRolloverThenFollowerRestart() throws Exception {
+ ZooKeeper zk = getClient(idxFollower);
+
+ int countCreated = createNodes(zk, 0, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxFollower);
+ start(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxFollower);
+ start(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxFollower);
+ start(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ // sanity check
+ assertTrue(countCreated > 0);
+ assertTrue(countCreated < 60);
+ }
+
+ /**
+ * Similar to testRolloverThenRestart, but ensure leadership can change,
+ * comes back, has the right data, and is able to serve new requests.
+ */
+ @Test
+ public void testRolloverThenLeaderRestart() throws Exception {
+ ZooKeeper zk = getClient(idxLeader);
+
+ int countCreated = createNodes(zk, 0, 10);
+
+ adjustEpochNearEnd();
+
+ checkNodes(zk, 0, countCreated);
+
+ shutdown(idxLeader);
+ start(idxLeader);
+ zk = getClient(idxLeader);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxLeader);
+ start(idxLeader);
+ zk = getClient(idxLeader);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxLeader);
+ start(idxLeader);
+ zk = getClient(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ // sanity check
+ assertTrue(countCreated > 0);
+ assertTrue(countCreated < 50);
+ }
+
+ /**
+ * Similar to testRolloverThenRestart, but ensure we can survive multiple
+ * epoch rollovers between restarts.
+ */
+ @Test
+ public void testMultipleRollover() throws Exception {
+ ZooKeeper zk = getClient(idxFollower);
+
+ int countCreated = createNodes(zk, 0, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ adjustEpochNearEnd();
+
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdownAll();
+ startAll();
+ zk = getClient(idxFollower);
+
+ adjustEpochNearEnd();
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ shutdown(idxLeader);
+ start(idxLeader);
+ zk = getClient(idxFollower);
+
+ checkNodes(zk, 0, countCreated);
+ countCreated += createNodes(zk, countCreated, 10);
+
+ // sanity check
+ assertTrue(countCreated > 0);
+ assertTrue(countCreated < 70);
+ }
+}
Modified: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/ClientBase.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/ClientBase.java?rev=1301082&r1=1301081&r2=1301082&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/ClientBase.java (original)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/ClientBase.java Thu Mar 15 16:50:16 2012
@@ -85,7 +85,7 @@ public abstract class ClientBase extends
public void process(WatchedEvent event) { /* nada */ }
}
- protected static class CountdownWatcher implements Watcher {
+ public static class CountdownWatcher implements Watcher {
// XXX this doesn't need to be volatile! (Should probably be final)
volatile CountDownLatch clientConnected;
volatile boolean connected;
@@ -108,10 +108,12 @@ public abstract class ClientBase extends
notifyAll();
}
}
- synchronized boolean isConnected() {
+ synchronized public boolean isConnected() {
return connected;
}
- synchronized void waitForConnected(long timeout) throws InterruptedException, TimeoutException {
+ synchronized public void waitForConnected(long timeout)
+ throws InterruptedException, TimeoutException
+ {
long expire = System.currentTimeMillis() + timeout;
long left = timeout;
while(!connected && left > 0) {
@@ -123,7 +125,9 @@ public abstract class ClientBase extends
}
}
- synchronized void waitForDisconnected(long timeout) throws InterruptedException, TimeoutException {
+ synchronized public void waitForDisconnected(long timeout)
+ throws InterruptedException, TimeoutException
+ {
long expire = System.currentTimeMillis() + timeout;
long left = timeout;
while(connected && left > 0) {