You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@zookeeper.apache.org by GitBox <gi...@apache.org> on 2019/01/12 02:46:35 UTC
[GitHub] jhuan31 closed pull request #769: ZOOKEEPER-3242: Add server side
connecting throttling
jhuan31 closed pull request #769: ZOOKEEPER-3242: Add server side connecting throttling
URL: https://github.com/apache/zookeeper/pull/769
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/BlueThrottle.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/BlueThrottle.java
new file mode 100644
index 0000000000..cc22f57284
--- /dev/null
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/BlueThrottle.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import java.util.Random;
+
+import org.apache.zookeeper.common.Time;
+
+/**
+ * Implements a token-bucket based rate limiting mechanism with optional
+ * probabilistic dropping inspired by the BLUE queue management algorithm [1].
+ *
+ * The throttle provides the {@link #checkLimit(int)} method which provides
+ * a binary yes/no decision.
+ *
+ * The core token bucket algorithm starts with an initial set of tokens based
+ * on the <code>maxTokens</code> setting. Tokens are dispensed each
+ * {@link #checkLimit(int)} call, which fails if there are not enough tokens to
+ * satisfy a given request.
+ *
+ * The token bucket refills over time, providing <code>fillCount</code> tokens
+ * every <code>fillTime</code> milliseconds, capping at <code>maxTokens</code>.
+ *
+ * This design allows the throttle to allow short bursts to pass, while still
+ * capping the total number of requests per time interval.
+ *
+ * One issue with a pure token bucket approach for something like request or
+ * connection throttling is that the wall clock arrival time of requests affects
+ * the probability of a request being allowed to pass or not. Under constant
+ * load this can lead to request starvation for requests that constantly arrive
+ * later than the majority.
+ *
+ * In an attempt to combat this, this throttle can also provide probabilistic
+ * dropping. This is enabled anytime <code>freezeTime</code> is set to a value
+ * other than <code>-1</code>.
+ *
+ * The probabilistic algorithm starts with an initial drop probability of 0, and
+ * adjusts this probability roughly every <code>freezeTime</code> milliseconds.
+ * The first request after <code>freezeTime</code>, the algorithm checks the
+ * token bucket. If the token bucket is empty, the drop probability is increased
+ * by <code>dropIncrease</code> up to a maximum of <code>1</code>. Otherwise, if
+ * the bucket has a token deficit less than <code>decreasePoint * maxTokens</code>,
+ * the probability is decreased by <code>dropDecrease</code>.
+ *
+ * Given a call to {@link #checkLimit(int)}, requests are first dropped randomly
+ * based on the current drop probability, and only surviving requests are then
+ * checked against the token bucket.
+ *
+ * When under constant load, the probabilistic algorithm will adapt to a drop
+ * frequency that should keep requests within the token limit. When load drops,
+ * the drop probability will decrease, eventually returning to zero if possible.
+ *
+ * [1] "BLUE: A New Class of Active Queue Management Algorithms"
+ **/
+
+public class BlueThrottle {
+ private int maxTokens;
+ private int fillTime;
+ private int fillCount;
+ private int tokens;
+ private long lastTime;
+
+ private int freezeTime;
+ private long lastFreeze;
+ private double dropIncrease;
+ private double dropDecrease;
+ private double decreasePoint;
+ private double drop;
+
+ Random rng;
+
+ public BlueThrottle() {
+ // Disable throttling by default (maxTokens = 0)
+ this.maxTokens = 0;
+ this.fillTime = 1;
+ this.fillCount = 1;
+ this.tokens = maxTokens;
+ this.lastTime = Time.currentElapsedTime();
+
+ // Disable BLUE throttling by default (freezeTime = -1)
+ this.freezeTime = -1;
+ this.lastFreeze = Time.currentElapsedTime();
+ this.dropIncrease = 0.02;
+ this.dropDecrease = 0.002;
+ this.decreasePoint = 0;
+ this.drop = 0;
+
+ this.rng = new Random();
+ }
+
+ public synchronized void setMaxTokens(int max) {
+ int deficit = maxTokens - tokens;
+ maxTokens = max;
+ tokens = max - deficit;
+ }
+
+ public synchronized void setFillTime(int time) {
+ fillTime = time;
+ }
+
+ public synchronized void setFillCount(int count) {
+ fillCount = count;
+ }
+
+ public synchronized void setFreezeTime(int time) {
+ freezeTime = time;
+ }
+
+ public synchronized void setDropIncrease(double increase) {
+ dropIncrease = increase;
+ }
+
+ public synchronized void setDropDecrease(double decrease) {
+ dropDecrease = decrease;
+ }
+
+ public synchronized void setDecreasePoint(double ratio) {
+ decreasePoint = ratio;
+ }
+
+ public synchronized int getMaxTokens() {
+ return maxTokens;
+ }
+
+ public synchronized int getFillTime() {
+ return fillTime;
+ }
+
+ public synchronized int getFillCount() {
+ return fillCount;
+ }
+
+ public synchronized int getFreezeTime() {
+ return freezeTime;
+ }
+
+ public synchronized double getDropIncrease() {
+ return dropIncrease;
+ }
+
+ public synchronized double getDropDecrease() {
+ return dropDecrease;
+ }
+
+ public synchronized double getDecreasePoint() {
+ return decreasePoint;
+ }
+
+ public double getDropChance() {
+ return drop;
+ }
+
+ public int getDeficit() {
+ return maxTokens - tokens;
+ }
+
+ public synchronized boolean checkLimit(int need) {
+ // A maxTokens setting of zero disables throttling
+ if (maxTokens == 0)
+ return true;
+
+ long now = Time.currentElapsedTime();
+ long diff = now - lastTime;
+
+ if (diff > fillTime) {
+ int refill = (int)(diff * fillCount / fillTime);
+ tokens = Math.min(tokens + refill, maxTokens);
+ lastTime = now;
+ }
+
+ // A freeze time of -1 disables BLUE randomized throttling
+ if(freezeTime != -1) {
+ if(!checkBlue(now)) {
+ return false;
+ }
+ }
+
+ if (tokens < need) {
+ return false;
+ }
+
+ tokens -= need;
+ return true;
+ }
+
+ public boolean checkBlue(long now) {
+ int length = maxTokens - tokens;
+ int limit = maxTokens;
+ long diff = now - lastFreeze;
+ long threshold = Math.round(maxTokens * decreasePoint);
+
+ if (diff > freezeTime) {
+ if((length == limit) && (drop < 1)) {
+ drop += dropIncrease;
+ }
+ else if ((length <= threshold) && (drop >= dropDecrease)) {
+ drop -= dropDecrease;
+ }
+ lastFreeze = now;
+ }
+
+ if (rng.nextDouble() <= drop) {
+ return false;
+ }
+ return true;
+ }
+};
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ClientCnxnLimitException.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ClientCnxnLimitException.java
new file mode 100644
index 0000000000..38f89959a5
--- /dev/null
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ClientCnxnLimitException.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+/**
+ * Indicates that the number of client connections has exceeded some limit.
+ * @see org.apache.zookeeper.server.ClientCnxnLimit#checkLimit()
+ * @see org.apache.zookeeper.server.ClientCnxnLimit#checkLimit(int)
+ */
+public class ClientCnxnLimitException extends Exception {
+ public ClientCnxnLimitException() {
+ super("Connection throttle rejected connection");
+ }
+}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
index b48eb3dc3b..a7cbd2df94 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java
@@ -147,7 +147,7 @@ public void sendBuffer(ByteBuffer bb) {
}
/** Read the request payload (everything following the length prefix) */
- private void readPayload() throws IOException, InterruptedException {
+ private void readPayload() throws IOException, InterruptedException, ClientCnxnLimitException {
if (incomingBuffer.remaining() != 0) { // have we read length bytes?
int rc = sock.read(incomingBuffer); // sock is non-blocking, so ok
if (rc < 0) {
@@ -352,6 +352,14 @@ void doIO(SelectionKey k) throws InterruptedException {
LOG.warn(e.getMessage());
// expecting close to log session closure
close();
+ } catch (ClientCnxnLimitException e) {
+ // Common case exception, print at debug level
+ ServerMetrics.CONNECTION_REJECTED.add(1);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exception causing close of session 0x"
+ + Long.toHexString(sessionId) + ": " + e.getMessage());
+ }
+ close();
} catch (IOException e) {
LOG.warn("Exception causing close of session 0x"
+ Long.toHexString(sessionId) + ": " + e.getMessage());
@@ -399,7 +407,7 @@ public void enableRecv() {
}
}
- private void readConnectRequest() throws IOException, InterruptedException {
+ private void readConnectRequest() throws IOException, InterruptedException, ClientCnxnLimitException {
if (!isZKServerRunning()) {
throw new IOException("ZooKeeperServer not running");
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
index 4e7e5db45f..090ee7b2c7 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java
@@ -310,6 +310,7 @@ private boolean doAccept() {
acceptErrorLogger.flush();
} catch (IOException e) {
// accept, maxClientCnxns, configureBlocking
+ ServerMetrics.CONNECTION_REJECTED.add(1);
acceptErrorLogger.rateLimitLog(
"Error accepting new connection: " + e.getMessage());
fastCloseSock(sc);
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
index 311d3c1d20..054dea1db4 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java
@@ -483,6 +483,13 @@ private void receiveMessage(ByteBuf message) {
} catch(IOException e) {
LOG.warn("Closing connection to " + getRemoteSocketAddress(), e);
close();
+ } catch(ClientCnxnLimitException e) {
+ // Common case exception, print at debug level
+ ServerMetrics.CONNECTION_REJECTED.add(1);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Closing connection to " + getRemoteSocketAddress(), e);
+ }
+ close();
}
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
index d96f56de9f..e0d55a4fbc 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java
@@ -108,6 +108,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception {
InetAddress addr = ((InetSocketAddress) channel.remoteAddress())
.getAddress();
if (maxClientCnxns > 0 && getClientCnxnCount(addr) >= maxClientCnxns) {
+ ServerMetrics.CONNECTION_REJECTED.add(1);
LOG.warn("Too many connections from {} - max is {}", addr,
maxClientCnxns);
channel.close();
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java
index c5d82deebb..185566f38d 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java
@@ -67,6 +67,11 @@
SNAP_COUNT(new SimpleCounter("snap_count")),
COMMIT_COUNT(new SimpleCounter("commit_count")),
CONNECTION_REQUEST_COUNT(new SimpleCounter("connection_request_count")),
+
+ // Connection throttling related
+ CONNECTION_TOKEN_DEFICIT(new AvgMinMaxCounter("connection_token_deficit")),
+ CONNECTION_REJECTED(new SimpleCounter("connection_rejected")),
+
BYTES_RECEIVED_COUNT(new SimpleCounter("bytes_received_count"));
private final Metric metric;
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
index 20ab023ec5..3d77e237c4 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -138,6 +138,9 @@
private ZooKeeperServerShutdownHandler zkShutdownHandler;
private volatile int createSessionTrackerServerId = 1;
+ // Connection throttling
+ private BlueThrottle connThrottle;
+
void removeCnxn(ServerCnxn cnxn) {
zkDb.removeCnxn(cnxn);
}
@@ -170,6 +173,30 @@ public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime,
setMinSessionTimeout(minSessionTimeout);
setMaxSessionTimeout(maxSessionTimeout);
listener = new ZooKeeperServerListenerImpl(this);
+
+ connThrottle = new BlueThrottle();
+ connThrottle.setMaxTokens(
+ Integer.getInteger("zookeeper.connection_throttle_tokens", 0)
+ );
+ connThrottle.setFillTime(
+ Integer.getInteger("zookeeper.connection_throttle_fill_time", 1)
+ );
+ connThrottle.setFillCount(
+ Integer.getInteger("zookeeper.connection_throttle_fill_count", 1)
+ );
+ connThrottle.setFreezeTime(
+ Integer.getInteger("zookeeper.connection_throttle_freeze_time", -1)
+ );
+ connThrottle.setDropIncrease(
+ getDoubleProp("zookeeper.connection_throttle_drop_increase", 0.02)
+ );
+ connThrottle.setDropDecrease(
+ getDoubleProp("zookeeper.connection_throttle_drop_decrease", 0.002)
+ );
+ connThrottle.setDecreasePoint(
+ getDoubleProp("zookeeper.connection_throttle_decrease_ratio", 0)
+ );
+
LOG.info("Created server with tickTime " + tickTime
+ " minSessionTimeout " + getMinSessionTimeout()
+ " maxSessionTimeout " + getMaxSessionTimeout()
@@ -177,6 +204,17 @@ public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime,
+ " snapdir " + txnLogFactory.getSnapDir());
}
+ /* Varation of Integer.getInteger for real number properties */
+ double getDoubleProp(String name, double def) {
+ String val = System.getProperty(name);
+ if(val != null) {
+ return Double.parseDouble(val);
+ }
+ else {
+ return def;
+ }
+ }
+
/**
* creates a zookeeperserver instance.
* @param txnLogFactory the file transaction snapshot logging class
@@ -192,6 +230,10 @@ public ServerStats serverStats() {
return serverStats;
}
+ public BlueThrottle connThrottle() {
+ return connThrottle;
+ }
+
public void dumpConf(PrintWriter pwriter) {
pwriter.print("clientPort=");
pwriter.println(getClientPort());
@@ -1016,7 +1058,18 @@ public void dumpEphemerals(PrintWriter pwriter) {
return zkDb.getEphemerals();
}
- public void processConnectRequest(ServerCnxn cnxn, ByteBuffer incomingBuffer) throws IOException {
+ public double getConnectionDropChance() {
+ return connThrottle.getDropChance();
+ }
+
+ public void processConnectRequest(ServerCnxn cnxn, ByteBuffer incomingBuffer)
+ throws IOException, ClientCnxnLimitException {
+
+ if (connThrottle.checkLimit(1) == false) {
+ throw new ClientCnxnLimitException();
+ }
+ ServerMetrics.CONNECTION_TOKEN_DEFICIT.add(connThrottle.getDeficit());
+
BinaryInputArchive bia = BinaryInputArchive.getArchive(new ByteBufferInputStream(incomingBuffer));
ConnectRequest connReq = new ConnectRequest();
connReq.deserialize(bia, "connect");
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerBean.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerBean.java
index cf84b2f9e5..e905a5dcd1 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerBean.java
@@ -167,7 +167,7 @@ public String getSecureClientPort() {
public String getSecureClientAddress() {
if (zks.secureServerCnxnFactory != null) {
return String.format("%s:%d", zks.secureServerCnxnFactory
- .getLocalAddress().getHostString(),
+ .getLocalAddress().getHostString(),
zks.secureServerCnxnFactory.getLocalPort());
}
return "";
@@ -197,4 +197,75 @@ public int getMinClientResponseSize() {
public int getMaxClientResponseSize() {
return zks.serverStats().getClientResponseStats().getMaxBufferSize();
}
+
+ // Connection throttling settings
+ ///////////////////////////////////////////////////////////////////////////
+
+ public int getConnectionMaxTokens() {
+ return zks.connThrottle().getMaxTokens();
+ }
+
+ public void setConnectionMaxTokens(int val) {
+ zks.connThrottle().setMaxTokens(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public int getConnectionTokenFillTime() {
+ return zks.connThrottle().getFillTime();
+ }
+
+ public void setConnectionTokenFillTime(int val) {
+ zks.connThrottle().setFillTime(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public int getConnectionTokenFillCount() {
+ return zks.connThrottle().getFillCount();
+ }
+
+ public void setConnectionTokenFillCount(int val) {
+ zks.connThrottle().setFillCount(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public int getConnectionFreezeTime() {
+ return zks.connThrottle().getFreezeTime();
+ }
+
+ public void setConnectionFreezeTime(int val) {
+ zks.connThrottle().setFreezeTime(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public double getConnectionDropIncrease() {
+ return zks.connThrottle().getDropIncrease();
+ }
+
+ public void setConnectionDropIncrease(double val) {
+ zks.connThrottle().setDropIncrease(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public double getConnectionDropDecrease() {
+ return zks.connThrottle().getDropDecrease();
+ }
+
+ public void setConnectionDropDecrease(double val) {
+ zks.connThrottle().setDropDecrease(val);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ public double getConnectionDecreaseRatio() {
+ return zks.connThrottle().getDecreasePoint();
+ }
+
+ public void setConnectionDecreaseRatio(double val) {
+ zks.connThrottle().setDecreasePoint(val);
+ }
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerMXBean.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
index feb6875870..a43a142567 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
@@ -95,6 +95,28 @@
*/
public void setMaxSessionTimeout(int max);
+ /* Connection throttling settings */
+ public int getConnectionMaxTokens();
+ public void setConnectionMaxTokens(int val);
+
+ public int getConnectionTokenFillTime();
+ public void setConnectionTokenFillTime(int val);
+
+ public int getConnectionTokenFillCount();
+ public void setConnectionTokenFillCount(int val);
+
+ public int getConnectionFreezeTime();
+ public void setConnectionFreezeTime(int val);
+
+ public double getConnectionDropIncrease();
+ public void setConnectionDropIncrease(double val);
+
+ public double getConnectionDropDecrease();
+ public void setConnectionDropDecrease(double val);
+
+ public double getConnectionDecreaseRatio();
+ public void setConnectionDecreaseRatio(double val);
+
/**
* Reset packet and latency statistics
*/
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
index f1e5500563..b0fa4ff010 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
@@ -354,6 +354,7 @@ public CommandResponse run(ZooKeeperServer zkServer, Map<String, String> kwargs)
OSMXBean osMbean = new OSMXBean();
response.put("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount());
response.put("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
+ response.put("connection_drop_probability", zkServer.getConnectionDropChance());
response.put("last_client_response_size", stats.getClientResponseStats().getLastBufferSize());
response.put("max_client_response_size", stats.getClientResponseStats().getMaxBufferSize());
diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
index 9b30c555dc..000b3cea41 100644
--- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
@@ -190,7 +190,8 @@ public void testMonitor() throws IOException, InterruptedException {
new Field("min_client_response_size", Integer.class),
new Field("uptime", Long.class),
new Field("global_sessions", Long.class),
- new Field("local_sessions", Long.class)
+ new Field("local_sessions", Long.class),
+ new Field("connection_drop_probability", Double.class)
));
for (String metric : ServerMetrics.getAllValues().keySet()) {
if (metric.startsWith("avg_")) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services