You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by fp...@apache.org on 2016/08/13 13:57:50 UTC
svn commit: r1756270 - in /zookeeper/branches/branch-3.5: ./
src/java/main/org/apache/zookeeper/server/
src/java/main/org/apache/zookeeper/server/quorum/
src/java/test/org/apache/zookeeper/server/
src/java/test/org/apache/zookeeper/test/
Author: fpj
Date: Sat Aug 13 13:57:49 2016
New Revision: 1756270
URL: http://svn.apache.org/viewvc?rev=1756270&view=rev
Log:
ZOOKEEPER-2247: Zookeeper service becomes unavailable when leader fails to write transaction log (Rakesh via fpj)
Added:
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
Modified:
zookeeper/branches/branch-3.5/CHANGES.txt
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
Modified: zookeeper/branches/branch-3.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/CHANGES.txt?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/CHANGES.txt (original)
+++ zookeeper/branches/branch-3.5/CHANGES.txt Sat Aug 13 13:57:49 2016
@@ -21,6 +21,9 @@ BUGFIXES:
(Abraham Fine via phunt)
Fix command handling in the C client shell (phunt via fpj)
+
+ ZOOKEEPER-2247: Zookeeper service becomes unavailable when leader
+ fails to write transaction log (Rakesh via fpj)
IMPROVEMENTS:
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java Sat Aug 13 13:57:49 2016
@@ -102,8 +102,8 @@ public class ZooKeeperServer implements
protected RequestProcessor firstProcessor;
protected volatile State state = State.INITIAL;
- enum State {
- INITIAL, RUNNING, SHUTDOWN;
+ protected enum State {
+ INITIAL, RUNNING, SHUTDOWN, ERROR;
}
/**
@@ -122,7 +122,9 @@ public class ZooKeeperServer implements
protected ServerCnxnFactory secureServerCnxnFactory;
private final ServerStats serverStats;
- private final ZooKeeperServerListener listener = new ZooKeeperServerListenerImpl();
+ private final ZooKeeperServerListener listener;
+ private ZooKeeperServerShutdownHandler zkShutdownHandler;
+
void removeCnxn(ServerCnxn cnxn) {
zkDb.removeCnxn(cnxn);
}
@@ -136,6 +138,7 @@ public class ZooKeeperServer implements
*/
public ZooKeeperServer() {
serverStats = new ServerStats(this);
+ listener = new ZooKeeperServerListenerImpl(this);
}
/**
@@ -152,7 +155,7 @@ public class ZooKeeperServer implements
this.tickTime = tickTime;
setMinSessionTimeout(minSessionTimeout);
setMaxSessionTimeout(maxSessionTimeout);
-
+ listener = new ZooKeeperServerListenerImpl(this);
LOG.info("Created server with tickTime " + tickTime
+ " minSessionTimeout " + getMinSessionTimeout()
+ " maxSessionTimeout " + getMaxSessionTimeout()
@@ -446,7 +449,7 @@ public class ZooKeeperServer implements
registerJMX();
- state = State.RUNNING;
+ setState(State.RUNNING);
notifyAll();
}
@@ -463,20 +466,6 @@ public class ZooKeeperServer implements
return listener;
}
- /**
- * Default listener implementation, which will do a graceful shutdown on
- * notification
- */
- private class ZooKeeperServerListenerImpl implements
- ZooKeeperServerListener {
-
- @Override
- public void notifyStopping(String threadName, int exitCode) {
- LOG.info("Thread {} exits, error code {}", threadName, exitCode);
- shutdown();
- }
- }
-
protected void createSessionTracker() {
sessionTracker = new SessionTrackerImpl(this, zkDb.getSessionWithTimeOuts(),
tickTime, 1, getZooKeeperServerListener());
@@ -486,19 +475,61 @@ public class ZooKeeperServer implements
((SessionTrackerImpl)sessionTracker).start();
}
+ /**
+ * Sets the state of ZooKeeper server. After changing the state, it notifies
+ * the server state change to a registered shutdown handler, if any.
+ * <p>
+ * The following are the server state transitions:
+ * <li>During startup the server will be in the INITIAL state.</li>
+ * <li>After successfully starting, the server sets the state to RUNNING.
+ * </li>
+ * <li>The server transitions to the ERROR state if it hits an internal
+ * error. {@link ZooKeeperServerListenerImpl} notifies any critical resource
+ * error events, e.g., SyncRequestProcessor not being able to write a txn to
+ * disk.</li>
+ * <li>During shutdown the server sets the state to SHUTDOWN, which
+ * corresponds to the server not running.</li>
+ *
+ * @param state new server state.
+ */
+ protected void setState(State state) {
+ this.state = state;
+ // Notify server state changes to the registered shutdown handler, if any.
+ if (zkShutdownHandler != null) {
+ zkShutdownHandler.handle(state);
+ } else {
+ LOG.error("ZKShutdownHandler is not registered, so ZooKeeper server "
+ + "won't take any action on ERROR or SHUTDOWN server state changes");
+ }
+ }
+
+ /**
+ * This can be used while shutting down the server to see whether the server
+ * is already shutdown or not.
+ *
+ * @return true if the server is running or server hits an error, false
+ * otherwise.
+ */
+ protected boolean canShutdown() {
+ return state == State.RUNNING || state == State.ERROR;
+ }
+
+ /**
+ * @return true if the server is running, false otherwise.
+ */
public boolean isRunning() {
return state == State.RUNNING;
}
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
LOG.info("shutting down");
// new RuntimeException("Calling shutdown").printStackTrace();
- state = State.SHUTDOWN;
+ setState(State.SHUTDOWN);
// Since sessionTracker and syncThreads poll we just have to
// set running to false and they will detect it during the poll
// interval.
@@ -1142,4 +1173,15 @@ public class ZooKeeperServer implements
return sessionTracker.getSessionExpiryMap();
}
+ /**
+ * This method is used to register the ZooKeeperServerShutdownHandler to get
+ * server's error or shutdown state change notifications.
+ * {@link ZooKeeperServerShutdownHandler#handle(State)} will be called for
+ * every server state changes {@link #setState(State)}.
+ *
+ * @param zkShutdownHandler shutdown handler
+ */
+ void registerServerShutdownHandler(ZooKeeperServerShutdownHandler zkShutdownHandler) {
+ this.zkShutdownHandler = zkShutdownHandler;
+ }
}
Added: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java?rev=1756270&view=auto
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java (added)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java Sat Aug 13 13:57:49 2016
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server;
+
+import org.apache.zookeeper.server.ZooKeeperServer.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default listener implementation, which will be used to notify internal
+ * errors. For example, if some critical thread has stopped due to fatal errors,
+ * then it will get notifications and will change the state of ZooKeeper server
+ * to ERROR representing an error status.
+ */
+class ZooKeeperServerListenerImpl implements ZooKeeperServerListener {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(ZooKeeperServerListenerImpl.class);
+
+ private final ZooKeeperServer zkServer;
+
+ ZooKeeperServerListenerImpl(ZooKeeperServer zkServer) {
+ this.zkServer = zkServer;
+ }
+
+ @Override
+ public void notifyStopping(String threadName, int exitCode) {
+ LOG.info("Thread {} exits, error code {}", threadName, exitCode);
+ zkServer.setState(State.ERROR);
+ }
+}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java Sat Aug 13 13:57:49 2016
@@ -19,6 +19,7 @@
package org.apache.zookeeper.server;
import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.management.JMException;
@@ -118,9 +119,15 @@ public class ZooKeeperServerMain {
// run() in this thread.
// create a file logger url from the command line args
txnLog = new FileTxnSnapLog(config.dataLogDir, config.dataDir);
- ZooKeeperServer zkServer = new ZooKeeperServer( txnLog,
+ final ZooKeeperServer zkServer = new ZooKeeperServer(txnLog,
config.tickTime, config.minSessionTimeout, config.maxSessionTimeout, null);
+ // Registers shutdown handler which will be used to know the
+ // server error or shutdown state changes.
+ final CountDownLatch shutdownLatch = new CountDownLatch(1);
+ zkServer.registerServerShutdownHandler(
+ new ZooKeeperServerShutdownHandler(shutdownLatch));
+
// Start Admin server
adminServer = AdminServerFactory.createAdminServer();
adminServer.setZooKeeperServer(zkServer);
@@ -146,14 +153,19 @@ public class ZooKeeperServerMain {
);
containerManager.start();
+ // Watch status of ZooKeeper server. It will do a graceful shutdown
+ // if the server is not running or hits an internal error.
+ shutdownLatch.await();
+
+ shutdown();
+
if (cnxnFactory != null) {
cnxnFactory.join();
}
if (secureCnxnFactory != null) {
secureCnxnFactory.join();
}
-
- if (zkServer.isRunning()) {
+ if (zkServer.canShutdown()) {
zkServer.shutdown();
}
} catch (InterruptedException e) {
@@ -180,9 +192,16 @@ public class ZooKeeperServerMain {
secureCnxnFactory.shutdown();
}
try {
- adminServer.shutdown();
+ if (adminServer != null) {
+ adminServer.shutdown();
+ }
} catch (AdminServerException e) {
LOG.warn("Problem stopping AdminServer", e);
}
}
+
+ // VisibleForTesting
+ ServerCnxnFactory getCnxnFactory() {
+ return cnxnFactory;
+ }
}
Added: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java?rev=1756270&view=auto
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java (added)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java Sat Aug 13 13:57:49 2016
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.zookeeper.server.ZooKeeperServer.State;
+
+/**
+ * ZooKeeper server shutdown handler which will be used to handle ERROR or
+ * SHUTDOWN server state transitions, which in turn releases the associated
+ * shutdown latch.
+ */
+class ZooKeeperServerShutdownHandler {
+ private final CountDownLatch shutdownLatch;
+
+ ZooKeeperServerShutdownHandler(CountDownLatch shutdownLatch) {
+ this.shutdownLatch = shutdownLatch;
+ }
+
+ /**
+ * This will be invoked when the server transition to a new server state.
+ *
+ * @param state new server state
+ */
+ void handle(State state) {
+ if (state == State.ERROR || state == State.SHUTDOWN) {
+ shutdownLatch.countDown();
+ }
+ }
+}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Follower.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Follower.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Follower.java Sat Aug 13 13:57:49 2016
@@ -85,7 +85,7 @@ public class Follower extends Learner{
}
syncWithLeader(newEpochZxid);
QuorumPacket qp = new QuorumPacket();
- while (self.isRunning()) {
+ while (this.isRunning()) {
readPacket(qp);
processPacket(qp);
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Leader.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Leader.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Leader.java Sat Aug 13 13:57:49 2016
@@ -586,6 +586,12 @@ public class Leader {
}
}
+ // check leader running status
+ if (!this.isRunning()) {
+ shutdown("Unexpected internal error");
+ return;
+ }
+
if (!tickSkip && !syncedAckSet.hasAllQuorums()) {
// Lost quorum of last committed and/or last proposed
// config, set shutdown flag
@@ -1380,4 +1386,8 @@ public class Leader {
return "UNKNOWN";
}
}
+
+ private boolean isRunning() {
+ return self.isRunning() && zk.isRunning();
+ }
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Learner.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Learner.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Learner.java Sat Aug 13 13:57:49 2016
@@ -622,4 +622,8 @@ public class Learner {
zk.shutdown();
}
}
+
+ boolean isRunning() {
+ return self.isRunning() && zk.isRunning();
+ }
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java Sat Aug 13 13:57:49 2016
@@ -158,7 +158,7 @@ public abstract class LearnerZooKeeperSe
@Override
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Observer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Observer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/Observer.java Sat Aug 13 13:57:49 2016
@@ -23,13 +23,8 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import org.apache.jute.Record;
-import org.apache.zookeeper.ZooDefs.OpCode;
import org.apache.zookeeper.server.ObserverBean;
import org.apache.zookeeper.server.Request;
-import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType;
-import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
-import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
import org.apache.zookeeper.server.util.SerializeUtils;
import org.apache.zookeeper.txn.SetDataTxn;
@@ -78,7 +73,7 @@ public class Observer extends Learner{
syncWithLeader(newLeaderZxid);
QuorumPacket qp = new QuorumPacket();
- while (self.isRunning()) {
+ while (this.isRunning()) {
readPacket(qp);
processPacket(qp);
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java Sat Aug 13 13:57:49 2016
@@ -132,7 +132,7 @@ public class ObserverZooKeeperServer ext
@Override
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java Sat Aug 13 13:57:49 2016
@@ -156,4 +156,9 @@ public abstract class QuorumZooKeeperSer
pwriter.println("membership: ");
pwriter.print(new String(self.getQuorumVerifier().toString().getBytes()));
}
+
+ @Override
+ protected void setState(State state) {
+ this.state = state;
+ }
}
Modified: zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java (original)
+++ zookeeper/branches/branch-3.5/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java Sat Aug 13 13:57:49 2016
@@ -137,7 +137,7 @@ public class ReadOnlyZooKeeperServer ext
@Override
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
@@ -172,4 +172,9 @@ public class ReadOnlyZooKeeperServer ext
pwriter.print("peerType=");
pwriter.println(self.getLearnerType().ordinal());
}
+
+ @Override
+ protected void setState(State state) {
+ this.state = state;
+ }
}
Modified: zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java?rev=1756270&r1=1756269&r2=1756270&view=diff
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java (original)
+++ zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java Sat Aug 13 13:57:49 2016
@@ -19,6 +19,7 @@
package org.apache.zookeeper.server;
import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileWriter;
@@ -124,6 +125,10 @@ public class ZooKeeperServerMainTest ext
throw new IOException("Failed to delete file: " + f);
}
}
+
+ ServerCnxnFactory getCnxnFactory() {
+ return main.getCnxnFactory();
+ }
}
public static class TestZKSMain extends ZooKeeperServerMain {
@@ -133,6 +138,63 @@ public class ZooKeeperServerMainTest ext
}
/**
+ * Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2247.
+ * Test to verify that even after non recoverable error (error while
+ * writing transaction log), ZooKeeper is still available.
+ */
+ @Test(timeout = 30000)
+ public void testNonRecoverableError() throws Exception {
+ ClientBase.setupTestEnv();
+
+ final int CLIENT_PORT = PortAssignment.unique();
+
+ MainThread main = new MainThread(CLIENT_PORT, true, null);
+ main.start();
+
+ Assert.assertTrue("waiting for server being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT,
+ CONNECTION_TIMEOUT));
+
+
+ ZooKeeper zk = new ZooKeeper("127.0.0.1:" + CLIENT_PORT,
+ ClientBase.CONNECTION_TIMEOUT, this);
+
+ zk.create("/foo1", "foobar".getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ Assert.assertEquals(new String(zk.getData("/foo1", null, null)), "foobar");
+
+ // inject problem in server
+ ZooKeeperServer zooKeeperServer = main.getCnxnFactory()
+ .getZooKeeperServer();
+ FileTxnSnapLog snapLog = zooKeeperServer.getTxnLogFactory();
+ FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(
+ snapLog.getDataDir(), snapLog.getSnapDir()) {
+ @Override
+ public void commit() throws IOException {
+ throw new IOException("Input/output error");
+ }
+ };
+ ZKDatabase newDB = new ZKDatabase(fileTxnSnapLogWithError);
+ zooKeeperServer.setZKDatabase(newDB);
+
+ try {
+ // do create operation, so that injected IOException is thrown
+ zk.create("/foo2", "foobar".getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ fail("IOException is expected as error is injected in transaction log commit funtionality");
+ } catch (Exception e) {
+ // do nothing
+ }
+ zk.close();
+ Assert.assertTrue("waiting for server down",
+ ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT,
+ ClientBase.CONNECTION_TIMEOUT));
+ fileTxnSnapLogWithError.close();
+ main.shutdown();
+ main.deleteDirs();
+ }
+
+ /**
* Verify the ability to start a standalone server instance.
*/
@Test
Added: zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java?rev=1756270&view=auto
==============================================================================
--- zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java (added)
+++ zookeeper/branches/branch-3.5/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java Sat Aug 13 13:57:49 2016
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.test;
+
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
+import org.apache.zookeeper.server.quorum.QuorumPeerTestBase;
+import org.apache.zookeeper.test.ClientBase.CountdownWatcher;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the non-recoverable error behavior of quorum server.
+ */
+public class NonRecoverableErrorTest extends QuorumPeerTestBase {
+ private static final String NODE_PATH = "/noLeaderIssue";
+
+ /**
+ * Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2247.
+ * Test to verify that even after non recoverable error (error while
+ * writing transaction log), ZooKeeper is still available.
+ */
+ @Test(timeout = 30000)
+ public void testZooKeeperServiceAvailableOnLeader() throws Exception {
+ int SERVER_COUNT = 3;
+ final int clientPorts[] = new int[SERVER_COUNT];
+ StringBuilder sb = new StringBuilder();
+ String server;
+
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ clientPorts[i] = PortAssignment.unique();
+ server = "server." + i + "=127.0.0.1:" + PortAssignment.unique()
+ + ":" + PortAssignment.unique() + ":participant;127.0.0.1:"
+ + clientPorts[i];
+ sb.append(server + "\n");
+ }
+ String currentQuorumCfgSection = sb.toString();
+ MainThread mt[] = new MainThread[SERVER_COUNT];
+
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection,
+ false);
+ mt[i].start();
+ }
+
+ // ensure server started
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ Assert.assertTrue("waiting for server " + i + " being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+ CONNECTION_TIMEOUT));
+ }
+
+ CountdownWatcher watcher = new CountdownWatcher();
+ ZooKeeper zk = new ZooKeeper("127.0.0.1:" + clientPorts[0],
+ ClientBase.CONNECTION_TIMEOUT, watcher);
+ watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT);
+
+ String data = "originalData";
+ zk.create(NODE_PATH, data.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+
+ // get information of current leader
+ QuorumPeer leader = getLeaderQuorumPeer(mt);
+ assertNotNull("Leader must have been elected by now", leader);
+
+ // inject problem in leader
+ FileTxnSnapLog snapLog = leader.getActiveServer().getTxnLogFactory();
+ FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(
+ snapLog.getDataDir(), snapLog.getSnapDir()) {
+ @Override
+ public void commit() throws IOException {
+ throw new IOException("Input/output error");
+ }
+ };
+ ZKDatabase originalZKDatabase = leader.getActiveServer()
+ .getZKDatabase();
+ long leaderCurrentEpoch = leader.getCurrentEpoch();
+
+ ZKDatabase newDB = new ZKDatabase(fileTxnSnapLogWithError);
+ leader.getActiveServer().setZKDatabase(newDB);
+
+ try {
+ // do create operation, so that injected IOException is thrown
+ zk.create(uniqueZnode(), data.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ fail("IOException is expected due to error injected to transaction log commit");
+ } catch (Exception e) {
+ // do nothing
+ }
+
+ // resetting watcher so that this watcher can be again used to ensure
+ // that the zkClient is able to re-establish connection with the
+ // newly elected zookeeper quorum.
+ watcher.reset();
+ waitForNewLeaderElection(leader, leaderCurrentEpoch);
+
+ // ensure server started, give enough time, so that new leader election
+ // takes place
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ Assert.assertTrue("waiting for server " + i + " being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+ CONNECTION_TIMEOUT));
+ }
+
+ // revert back the error
+ leader.getActiveServer().setZKDatabase(originalZKDatabase);
+
+ // verify that now ZooKeeper service is up and running
+ leader = getLeaderQuorumPeer(mt);
+ assertNotNull("New leader must have been elected by now", leader);
+
+ String uniqueNode = uniqueZnode();
+ watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT);
+ String createNode = zk.create(uniqueNode, data.getBytes(),
+ Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ // if node is created successfully then it means that ZooKeeper service
+ // is available
+ assertEquals("Failed to create znode", uniqueNode, createNode);
+ zk.close();
+ // stop all severs
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ mt[i].shutdown();
+ }
+ }
+
+ private void waitForNewLeaderElection(QuorumPeer peer,
+ long leaderCurrentEpoch) throws IOException, InterruptedException {
+ LOG.info("Waiting for new LE cycle..");
+ int count = 100; // giving a grace period of 10seconds
+ while (count > 0) {
+ if (leaderCurrentEpoch == peer.getCurrentEpoch()) {
+ Thread.sleep(100);
+ }
+ count--;
+ }
+ Assert.assertNotEquals("New LE cycle must have triggered",
+ leaderCurrentEpoch, peer.getCurrentEpoch());
+ }
+
+ private QuorumPeer getLeaderQuorumPeer(MainThread[] mt) {
+ for (int i = mt.length - 1; i >= 0; i--) {
+ QuorumPeer quorumPeer = mt[i].getQuorumPeer();
+ if (null != quorumPeer
+ && ServerState.LEADING == quorumPeer.getPeerState()) {
+ return quorumPeer;
+ }
+ }
+ return null;
+ }
+
+ private String uniqueZnode() {
+ UUID randomUUID = UUID.randomUUID();
+ String node = NODE_PATH + "/" + randomUUID.toString();
+ return node;
+ }
+}