You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by ha...@apache.org on 2019/07/02 22:44:46 UTC
[zookeeper] branch master updated: ZOOKEEPER-3439: Observability
improvements on client / server connection close.
This is an automated email from the ASF dual-hosted git repository.
hanm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 9bee98b ZOOKEEPER-3439: Observability improvements on client / server connection close.
9bee98b is described below
commit 9bee98bd06f50a2085785406a1084f75c652ada1
Author: Michael Han <lh...@twitter.com>
AuthorDate: Tue Jul 2 15:44:25 2019 -0700
ZOOKEEPER-3439: Observability improvements on client / server connection close.
Currently when server closes a client connection there is not enough information recorded (except few exception logs) which makes it hard to do postmortems. On the other side, having a complete view of the aggregated connection closing reason will provide more signals based on which we can better operate the clusters (e.g. predicate an incident might happen based on the trending of the connection closing reasons).
Server metrics was not added in this PR as we internally use a different metrics system - so some work needed to migrate to ServerMetrics. Want to submit first to get community feedback on this.
Author: Michael Han <lh...@twitter.com>
Reviewers: Enrico Olivelli <eo...@gmail.com>, maoling <ma...@sina.com>, enixon
Closes #997 from hanm/twitter/cc6d87505d9745caace2c86acf58d6ffa085281e
---
.../org/apache/zookeeper/server/DumbWatcher.java | 2 +-
.../zookeeper/server/FinalRequestProcessor.java | 2 +-
.../org/apache/zookeeper/server/NIOServerCnxn.java | 31 ++++++++----
.../zookeeper/server/NIOServerCnxnFactory.java | 16 +++---
.../apache/zookeeper/server/NettyServerCnxn.java | 14 ++++--
.../zookeeper/server/NettyServerCnxnFactory.java | 18 +++----
.../org/apache/zookeeper/server/ServerCnxn.java | 58 ++++++++++++++++++++--
.../apache/zookeeper/server/ServerCnxnFactory.java | 6 +--
.../apache/zookeeper/server/ZooKeeperServer.java | 12 ++---
.../auth/EnsembleAuthenticationProvider.java | 2 +-
.../apache/zookeeper/server/quorum/QuorumPeer.java | 5 +-
.../apache/zookeeper/server/MockServerCnxn.java | 2 +-
.../apache/zookeeper/server/NIOServerCnxnTest.java | 2 +-
.../apache/zookeeper/server/quorum/ZabUtils.java | 6 ++-
14 files changed, 123 insertions(+), 53 deletions(-)
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DumbWatcher.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DumbWatcher.java
index d83763b..ba8e57a 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DumbWatcher.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DumbWatcher.java
@@ -54,7 +54,7 @@ public class DumbWatcher extends ServerCnxn {
int getSessionTimeout() { return 0; }
@Override
- public void close() { }
+ public void close(DisconnectReason reason) { }
@Override
public void sendResponse(ReplyHeader h, Record r, String tag, String cacheKey, Stat stat) throws IOException { }
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/FinalRequestProcessor.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/FinalRequestProcessor.java
index 81f1145..e15d189 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/FinalRequestProcessor.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/FinalRequestProcessor.java
@@ -622,7 +622,7 @@ public class FinalRequestProcessor implements RequestProcessor {
if (serverCnxnFactory == null) {
return false;
}
- return serverCnxnFactory.closeSession(sessionId);
+ return serverCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_CLOSED_SESSION);
}
private boolean connClosedByClient(Request request) {
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
index c8ee076..fb7ce4d 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
@@ -159,7 +159,8 @@ public class NIOServerCnxn extends ServerCnxn {
throw new EndOfStreamException(
"Unable to read additional data from client sessionid 0x"
+ Long.toHexString(sessionId)
- + ", likely client has closed socket");
+ + ", likely client has closed socket",
+ DisconnectReason.UNABLE_TO_READ_FROM_CLIENT);
}
}
@@ -224,7 +225,8 @@ public class NIOServerCnxn extends ServerCnxn {
ByteBuffer bb;
while ((bb = outgoingBuffers.peek()) != null) {
if (bb == ServerCnxnFactory.closeConn) {
- throw new CloseRequestException("close requested");
+ throw new CloseRequestException("close requested",
+ DisconnectReason.CLIENT_CLOSED_CONNECTION);
}
if (bb == packetSentinel) {
packetSent();
@@ -274,7 +276,8 @@ public class NIOServerCnxn extends ServerCnxn {
// Remove the buffers that we have sent
while ((bb = outgoingBuffers.peek()) != null) {
if (bb == ServerCnxnFactory.closeConn) {
- throw new CloseRequestException("close requested");
+ throw new CloseRequestException("close requested",
+ DisconnectReason.CLIENT_CLOSED_CONNECTION);
}
if (bb == packetSentinel) {
packetSent();
@@ -319,7 +322,8 @@ public class NIOServerCnxn extends ServerCnxn {
throw new EndOfStreamException(
"Unable to read additional data from client sessionid 0x"
+ Long.toHexString(sessionId)
- + ", likely client has closed socket");
+ + ", likely client has closed socket",
+ DisconnectReason.UNABLE_TO_READ_FROM_CLIENT);
}
if (incomingBuffer.remaining() == 0) {
boolean isPayload;
@@ -345,7 +349,8 @@ public class NIOServerCnxn extends ServerCnxn {
handleWrite(k);
if (!initialized && !getReadInterest() && !getWriteInterest()) {
- throw new CloseRequestException("responded to info probe");
+ throw new CloseRequestException("responded to info probe",
+ DisconnectReason.INFO_PROBE);
}
}
} catch (CancelledKeyException e) {
@@ -354,14 +359,14 @@ public class NIOServerCnxn extends ServerCnxn {
if (LOG.isDebugEnabled()) {
LOG.debug("CancelledKeyException stack trace", e);
}
- close();
+ close(DisconnectReason.CANCELLED_KEY_EXCEPTION);
} catch (CloseRequestException e) {
// expecting close to log session closure
close();
} catch (EndOfStreamException e) {
LOG.warn(e.getMessage());
// expecting close to log session closure
- close();
+ close(e.getReason());
} catch (ClientCnxnLimitException e) {
// Common case exception, print at debug level
ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
@@ -369,14 +374,14 @@ public class NIOServerCnxn extends ServerCnxn {
LOG.debug("Exception causing close of session 0x"
+ Long.toHexString(sessionId) + ": " + e.getMessage());
}
- close();
+ close(DisconnectReason.CLIENT_CNX_LIMIT);
} catch (IOException e) {
LOG.warn("Exception causing close of session 0x"
+ Long.toHexString(sessionId) + ": " + e.getMessage());
if (LOG.isDebugEnabled()) {
LOG.debug("IOException stack trace", e);
}
- close();
+ close(DisconnectReason.IO_EXCEPTION);
}
}
@@ -576,11 +581,17 @@ public class NIOServerCnxn extends ServerCnxn {
" sessionId: 0x" + Long.toHexString(sessionId);
}
+
/**
* Close the cnxn and remove it from the factory cnxns list.
*/
@Override
- public void close() {
+ public void close(DisconnectReason reason) {
+ disconnectReason = reason;
+ close();
+ }
+
+ private void close() {
setStale();
if (!factory.removeCnxn(this)) {
return;
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
index 289c7a2..67d799b 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
@@ -404,7 +404,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
for (SelectionKey key : selector.keys()) {
NIOServerCnxn cnxn = (NIOServerCnxn) key.attachment();
if (cnxn.isSelectable()) {
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
}
cleanupSelectionKey(key);
}
@@ -532,7 +532,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
// Check if we shutdown or doIO() closed this connection
if (stopped) {
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
return;
}
if (!key.isValid()) {
@@ -548,13 +548,13 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
// on the current set of interest ops, which may have changed
// as a result of the I/O operations we just performed.
if (!selectorThread.addInterestOpsUpdateRequest(key)) {
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.CONNECTION_MODE_CHANGED);
}
}
@Override
public void cleanup() {
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.CLEAN_UP);
}
}
@@ -577,7 +577,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
}
for (NIOServerCnxn conn : cnxnExpiryQueue.poll()) {
ServerMetrics.getMetrics().SESSIONLESS_CONNECTIONS_EXPIRED.add(1);
- conn.close();
+ conn.close(ServerCnxn.DisconnectReason.CONNECTION_EXPIRED);
}
}
@@ -867,12 +867,12 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
*/
@Override
@SuppressWarnings("unchecked")
- public void closeAll() {
+ public void closeAll(ServerCnxn.DisconnectReason reason) {
// clear all the connections on which we are selecting
for (ServerCnxn cnxn : cnxns) {
try {
// This will remove the cnxn from cnxns
- cnxn.close();
+ cnxn.close(reason);
} catch (Exception e) {
LOG.warn("Ignoring exception closing cnxn sessionid 0x"
+ Long.toHexString(cnxn.getSessionId()), e);
@@ -921,7 +921,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
join();
// close all open connections
- closeAll();
+ closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
if (login != null) {
login.shutdown();
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
index c73af01..3a3442f 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
@@ -80,7 +80,15 @@ public class NettyServerCnxn extends ServerCnxn {
addAuthInfo(new Id("ip", addr.getHostAddress()));
}
+ /**
+ * Close the cnxn and remove it from the factory cnxns list.
+ */
@Override
+ public void close(DisconnectReason reason) {
+ disconnectReason = reason;
+ close();
+ }
+
public void close() {
closingChannel = true;
@@ -194,7 +202,7 @@ public class NettyServerCnxn extends ServerCnxn {
@Override
public void sendBuffer(ByteBuffer... buffers) {
if (buffers.length == 1 && buffers[0] == ServerCnxnFactory.closeConn) {
- close();
+ close(DisconnectReason.CLIENT_CLOSED_CONNECTION);
return;
}
channel.writeAndFlush(Unpooled.wrappedBuffer(buffers)).addListener(onSendBufferDoneListener);
@@ -533,14 +541,14 @@ public class NettyServerCnxn extends ServerCnxn {
}
} catch(IOException e) {
LOG.warn("Closing connection to " + getRemoteSocketAddress(), e);
- close();
+ close(DisconnectReason.IO_EXCEPTION);
} catch(ClientCnxnLimitException e) {
// Common case exception, print at debug level
ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
if (LOG.isDebugEnabled()) {
LOG.debug("Closing connection to " + getRemoteSocketAddress(), e);
}
- close();
+ close(DisconnectReason.CLIENT_RATE_LIMIT);
}
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
index 549f7dc..9a05df6 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLException;
import javax.net.ssl.SSLPeerUnverifiedException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509KeyManager;
@@ -58,7 +57,6 @@ import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.ChannelGroupFuture;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.ssl.OpenSsl;
import io.netty.handler.ssl.OptionalSslHandler;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslHandler;
@@ -222,7 +220,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
if (LOG.isTraceEnabled()) {
LOG.trace("Channel inactive caused close {}", cnxn);
}
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_DISCONNECTED);
}
}
@@ -234,7 +232,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
if (LOG.isDebugEnabled()) {
LOG.debug("Closing {}", cnxn);
}
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_CLOSED_EXCEPTION);
}
}
@@ -354,7 +352,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
if (authProvider == null) {
LOG.error("X509 Auth provider not found: {}", authProviderProp);
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.AUTH_PROVIDER_NOT_FOUND);
return;
}
@@ -362,7 +360,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
authProvider.handleAuthentication(cnxn, null)) {
LOG.error("Authentication failed for session 0x{}",
Long.toHexString(cnxn.getSessionId()));
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.SASL_AUTH_FAILURE);
return;
}
}
@@ -373,7 +371,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
} else {
LOG.error("Unsuccessful handshake with session 0x{}",
Long.toHexString(cnxn.getSessionId()));
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.FAILED_HANDSHAKE);
}
}
}
@@ -471,7 +469,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
}
@Override
- public void closeAll() {
+ public void closeAll(ServerCnxn.DisconnectReason reason) {
if (LOG.isDebugEnabled()) {
LOG.debug("closeAll()");
}
@@ -480,7 +478,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
for (ServerCnxn cnxn : cnxns) {
try {
// This will remove the cnxn from cnxns
- cnxn.close();
+ cnxn.close(reason);
} catch (Exception e) {
LOG.warn("Ignoring exception closing cnxn sessionid 0x"
+ Long.toHexString(cnxn.getSessionId()), e);
@@ -559,7 +557,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
bossGroup.shutdownGracefully();
});
}
- closeAll();
+ closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
ChannelGroupFuture allChannelsCloseFuture = allChannels.close();
if (workerGroup != null) {
allChannelsCloseFuture.addListener(future -> {
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxn.java
index 96e08a6..6953e3a 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxn.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxn.java
@@ -78,6 +78,47 @@ public abstract class ServerCnxn implements Stats, Watcher {
*/
final ZooKeeperServer zkServer;
+ public enum DisconnectReason {
+ UNKNOWN("unknown"),
+ SERVER_SHUTDOWN("server_shutdown"),
+ CLOSE_ALL_CONNECTIONS_FORCED("close_all_connections_forced"),
+ CONNECTION_CLOSE_FORCED("connection_close_forced"),
+ CONNECTION_EXPIRED("connection_expired"),
+ CLIENT_CLOSED_CONNECTION("client_closed_connection"),
+ CLIENT_CLOSED_SESSION("client_closed_session"),
+ UNABLE_TO_READ_FROM_CLIENT("unable_to_read_from_client"),
+ NOT_READ_ONLY_CLIENT("not_read_only_client"),
+ CLIENT_ZXID_AHEAD("client_zxid_ahead"),
+ INFO_PROBE("info_probe"),
+ CLIENT_RECONNECT("client_reconnect"),
+ CANCELLED_KEY_EXCEPTION("cancelled_key_exception"),
+ IO_EXCEPTION("io_exception"),
+ IO_EXCEPTION_IN_SESSION_INIT("io_exception_in_session_init"),
+ BUFFER_UNDERFLOW_EXCEPTION("buffer_underflow_exception"),
+ SASL_AUTH_FAILURE("sasl_auth_failure"),
+ RESET_COMMAND("reset_command"),
+ CLOSE_CONNECTION_COMMAND("close_connection_command"),
+ CLEAN_UP("clean_up"),
+ CONNECTION_MODE_CHANGED("connection_mode_changed"),
+ // Below reasons are NettyServerCnxnFactory only
+ CHANNEL_DISCONNECTED("channel disconnected"),
+ CHANNEL_CLOSED_EXCEPTION("channel_closed_exception"),
+ AUTH_PROVIDER_NOT_FOUND("auth provider not found"),
+ FAILED_HANDSHAKE("Unsuccessful handshake"),
+ CLIENT_RATE_LIMIT("Client hits rate limiting threshold"),
+ CLIENT_CNX_LIMIT("Client hits connection limiting threshold");
+
+ String disconnectReason;
+
+ DisconnectReason(String reason) {
+ this.disconnectReason = reason;
+ }
+
+ public String toDisconnectReasonString() {
+ return disconnectReason;
+ }
+ }
+
public ServerCnxn(final ZooKeeperServer zkServer) {
this.zkServer = zkServer;
}
@@ -103,7 +144,7 @@ public abstract class ServerCnxn implements Stats, Watcher {
}
}
- public abstract void close();
+ public abstract void close(DisconnectReason reason);
public abstract void sendResponse(ReplyHeader h, Record r,
String tag, String cacheKey, Stat stat) throws IOException;
@@ -203,22 +244,28 @@ public abstract class ServerCnxn implements Stats, Watcher {
protected static class CloseRequestException extends IOException {
private static final long serialVersionUID = -7854505709816442681L;
+ private DisconnectReason reason;
- public CloseRequestException(String msg) {
+ public CloseRequestException(String msg, DisconnectReason reason) {
super(msg);
+ this.reason = reason;
}
+ public DisconnectReason getReason() { return reason; }
}
protected static class EndOfStreamException extends IOException {
private static final long serialVersionUID = -8255690282104294178L;
+ private DisconnectReason reason;
- public EndOfStreamException(String msg) {
+ public EndOfStreamException(String msg, DisconnectReason reason) {
super(msg);
+ this.reason = reason;
}
public String toString() {
return "EndOfStreamException: " + getMessage();
}
+ public DisconnectReason getReason() { return reason; }
}
public boolean isStale() {
@@ -263,8 +310,11 @@ public abstract class ServerCnxn implements Stats, Watcher {
protected long count;
protected long totalLatency;
+ protected long requestsProcessedCount;
+ protected DisconnectReason disconnectReason = DisconnectReason.UNKNOWN;
public synchronized void resetStats() {
+ disconnectReason = DisconnectReason.RESET_COMMAND;
packetsReceived.set(0);
packetsSent.set(0);
minLatency = Long.MAX_VALUE;
@@ -470,7 +520,7 @@ public abstract class ServerCnxn implements Stats, Watcher {
LOG.info("Error closing PrintWriter ", e);
} finally {
try {
- close();
+ close(DisconnectReason.CLOSE_CONNECTION_COMMAND);
} catch (Exception e) {
LOG.error("Error closing a command socket ", e);
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxnFactory.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxnFactory.java
index c76175e..983b1b5 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxnFactory.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxnFactory.java
@@ -73,11 +73,11 @@ public abstract class ServerCnxnFactory {
* @return true if the cnxn that contains the sessionId exists in this ServerCnxnFactory
* and it's closed. Otherwise false.
*/
- public boolean closeSession(long sessionId) {
+ public boolean closeSession(long sessionId, ServerCnxn.DisconnectReason reason) {
ServerCnxn cnxn = sessionMap.remove(sessionId);
if (cnxn != null) {
try {
- cnxn.close();
+ cnxn.close(reason);
} catch (Exception e) {
LOG.warn("exception during session close", e);
}
@@ -154,7 +154,7 @@ public abstract class ServerCnxnFactory {
}
}
- public abstract void closeAll();
+ public abstract void closeAll(ServerCnxn.DisconnectReason reason);
static public ServerCnxnFactory createFactory() throws IOException {
String serverCnxnFactoryName =
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
index 5117bd8..f5c770b 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -887,7 +887,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
} catch (Exception e) {
LOG.warn("Exception while establishing session, closing", e);
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.IO_EXCEPTION_IN_SESSION_INIT);
}
}
@@ -1165,7 +1165,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
String msg = "Refusing session request for not-read-only client "
+ cnxn.getRemoteSocketAddress();
LOG.info(msg);
- throw new CloseRequestException(msg);
+ throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.CLIENT_ZXID_AHEAD);
}
if (connReq.getLastZxidSeen() > zkDb.dataTree.lastProcessedZxid) {
String msg = "Refusing session request for client "
@@ -1177,7 +1177,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
+ " client must try another server";
LOG.info(msg);
- throw new CloseRequestException(msg);
+ throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.NOT_READ_ONLY_CLIENT);
}
int sessionTimeout = connReq.getTimeOut();
byte passwd[] = connReq.getPasswd();
@@ -1211,10 +1211,10 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
connReq.getTimeOut(),
cnxn.getRemoteSocketAddress());
if (serverCnxnFactory != null) {
- serverCnxnFactory.closeSession(sessionId);
+ serverCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT);
}
if (secureServerCnxnFactory != null) {
- secureServerCnxnFactory.closeSession(sessionId);
+ secureServerCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT);
}
cnxn.setSessionId(sessionId);
reopenSession(cnxn, sessionId, passwd, sessionTimeout);
@@ -1370,7 +1370,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
LOG.warn("Maintaining client connection despite SASL authentication failure.");
} else {
LOG.warn("Closing client connection due to SASL authentication failure.");
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.SASL_AUTH_FAILURE);
}
}
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/EnsembleAuthenticationProvider.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/EnsembleAuthenticationProvider.java
index 2017769..0e045b7 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/EnsembleAuthenticationProvider.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/EnsembleAuthenticationProvider.java
@@ -113,7 +113,7 @@ public class EnsembleAuthenticationProvider implements AuthenticationProvider {
* shutdown.
*/
ServerMetrics.getMetrics().ENSEMBLE_AUTH_FAIL.add(1);
- cnxn.close();
+ cnxn.close(ServerCnxn.DisconnectReason.FAILED_HANDSHAKE);
return KeeperException.Code.BADARGUMENTS;
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
index 8811875..f3217af 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
@@ -55,6 +55,7 @@ import org.apache.zookeeper.common.Time;
import org.apache.zookeeper.common.X509Exception;
import org.apache.zookeeper.jmx.MBeanRegistry;
import org.apache.zookeeper.jmx.ZKMBeanInfo;
+import org.apache.zookeeper.server.ServerCnxn;
import org.apache.zookeeper.server.ServerCnxnFactory;
import org.apache.zookeeper.server.ServerMetrics;
import org.apache.zookeeper.server.ZKDatabase;
@@ -1887,10 +1888,10 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
public void closeAllConnections() {
if (cnxnFactory != null) {
- cnxnFactory.closeAll();
+ cnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
}
if (secureCnxnFactory != null) {
- secureCnxnFactory.closeAll();
+ secureCnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
}
}
diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/MockServerCnxn.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/MockServerCnxn.java
index bf9aea4..eb10720 100644
--- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/MockServerCnxn.java
+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/MockServerCnxn.java
@@ -40,7 +40,7 @@ public class MockServerCnxn extends ServerCnxn {
}
@Override
- public void close() {
+ public void close(DisconnectReason reason) {
}
@Override
diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/NIOServerCnxnTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/NIOServerCnxnTest.java
index 4362b2c..8795785 100644
--- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/NIOServerCnxnTest.java
+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/NIOServerCnxnTest.java
@@ -63,7 +63,7 @@ public class NIOServerCnxnTest extends ClientBase {
serverFactory instanceof NIOServerCnxnFactory);
Iterable<ServerCnxn> connections = serverFactory.getConnections();
for (ServerCnxn serverCnxn : connections) {
- serverCnxn.close();
+ serverCnxn.close(ServerCnxn.DisconnectReason.CHANNEL_CLOSED_EXCEPTION);
try {
serverCnxn.toString();
} catch (Exception e) {
diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/ZabUtils.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/ZabUtils.java
index fada7bb..09bcde8 100644
--- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/ZabUtils.java
+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/ZabUtils.java
@@ -128,10 +128,12 @@ public class ZabUtils {
boolean secure) throws IOException {
}
- public boolean closeSession(long sessionId) {
+ @Override
+ public boolean closeSession(long sessionId, ServerCnxn.DisconnectReason reason) {
return false;
}
- public void closeAll() {
+ @Override
+ public void closeAll(ServerCnxn.DisconnectReason reason) {
}
@Override
public int getNumAliveConnections() {