You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@curator.apache.org by eo...@apache.org on 2023/02/21 13:28:49 UTC

[curator] branch master updated: CURATOR-535: Fix client port conflict due to untrustworthy random port allocation (#421)

This is an automated email from the ASF dual-hosted git repository.

eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/curator.git


The following commit(s) were added to refs/heads/master by this push:
     new 7e7c2079 CURATOR-535: Fix client port conflict due to untrustworthy random port allocation (#421)
7e7c2079 is described below

commit 7e7c2079357fbf10081906d7b783f3483bdecbe7
Author: Kezhu Wang <ke...@gmail.com>
AuthorDate: Tue Feb 21 21:28:41 2023 +0800

    CURATOR-535: Fix client port conflict due to untrustworthy random port allocation (#421)
    
    This commit tries to solve port conflict for `TestingServer` if port is
    unspecified(aka. `port <= 0`):
    * Set `InstanceSpec.port` to 0 if port is unspecified.
    * Save OS chosen bind port after started to maintain same port across
      restart.
    
    Ideally, it should be possible to bootstrap `TestingCluster`(with unspecified ports)
    too with help from `reconfig`. But there are difficulties since election port, quorum
    port were not designed to be bound to `0` in ZooKeeper. `TestingServer` should be
    enough for most cases.
    
    Users should resort to other solutions(eg. container) if they got bored
    by port conflict due to usages of `TestingCluster` and
    `TestingServer.restart`.
---
 .../java/org/apache/curator/test/InstanceSpec.java |   4 +
 .../apache/curator/test/QuorumConfigBuilder.java   |  23 ++++-
 .../curator/test/QuorumPeerConfigBuilder.java      |  52 ++++++++++
 .../apache/curator/test/TestingQuorumPeerMain.java |  30 ++----
 .../org/apache/curator/test/TestingServer.java     |  17 ++--
 .../apache/curator/test/TestingZooKeeperMain.java  |  28 ++----
 .../curator/test/TestingZooKeeperServer.java       |  17 +++-
 .../org/apache/curator/test/ZooKeeperMainFace.java |   9 +-
 .../test/ZooKeeperServerEmbeddedAdapter.java       | 108 ++++++++++++++++-----
 .../org/apache/curator/test/TestTestingServer.java |   5 +-
 10 files changed, 209 insertions(+), 84 deletions(-)

diff --git a/curator-test/src/main/java/org/apache/curator/test/InstanceSpec.java b/curator-test/src/main/java/org/apache/curator/test/InstanceSpec.java
index b1d80e6d..375d8801 100644
--- a/curator-test/src/main/java/org/apache/curator/test/InstanceSpec.java
+++ b/curator-test/src/main/java/org/apache/curator/test/InstanceSpec.java
@@ -205,6 +205,10 @@ public class InstanceSpec
         return quorumPort;
     }
 
+    /**
+     * @deprecated use {@link TestingServer#getConnectString()} or {@link TestingCluster#getConnectString()} instead
+     */
+    @Deprecated
     public String getConnectString()
     {
         return hostname + ":" + port;
diff --git a/curator-test/src/main/java/org/apache/curator/test/QuorumConfigBuilder.java b/curator-test/src/main/java/org/apache/curator/test/QuorumConfigBuilder.java
index 42f7b27b..35c3523e 100644
--- a/curator-test/src/main/java/org/apache/curator/test/QuorumConfigBuilder.java
+++ b/curator-test/src/main/java/org/apache/curator/test/QuorumConfigBuilder.java
@@ -97,7 +97,13 @@ public class QuorumConfigBuilder implements Closeable
 
     public QuorumPeerConfig buildConfig(int instanceIndex) throws Exception
     {
-        Properties properties = buildConfigProperties(instanceIndex);
+        InstanceSpec spec = instanceSpecs.get(instanceIndex);
+        return buildConfig(instanceIndex, spec.getPort());
+    }
+
+    public QuorumPeerConfig buildConfig(int instanceIndex, int instancePort) throws Exception
+    {
+        Properties properties = buildConfigProperties(instanceIndex, instancePort);
         QuorumPeerConfig config = new QuorumPeerConfig()
         {
             {
@@ -112,6 +118,12 @@ public class QuorumConfigBuilder implements Closeable
     }
 
     public Properties buildConfigProperties(int instanceIndex) throws Exception
+    {
+        InstanceSpec spec = instanceSpecs.get(instanceIndex);
+        return buildConfigProperties(instanceIndex, spec.getPort());
+    }
+
+    public Properties buildConfigProperties(int instanceIndex, int instancePort) throws Exception
     {
         boolean isCluster = (instanceSpecs.size() > 1);
         InstanceSpec spec = instanceSpecs.get(instanceIndex);
@@ -128,7 +140,7 @@ public class QuorumConfigBuilder implements Closeable
         properties.setProperty("initLimit", "10");
         properties.setProperty("syncLimit", "5");
         properties.setProperty("dataDir", spec.getDataDirectory().getCanonicalPath());
-        properties.setProperty("clientPort", Integer.toString(spec.getPort()));
+        properties.setProperty("clientPort", Integer.toString(instancePort));
         String tickTime = Integer.toString((spec.getTickTime() >= 0) ? spec.getTickTime() : new Timing2().tickTime());
         properties.setProperty("tickTime", tickTime);
         properties.setProperty("minSessionTimeout", tickTime);
@@ -142,7 +154,8 @@ public class QuorumConfigBuilder implements Closeable
         {
             for ( InstanceSpec thisSpec : instanceSpecs )
             {
-                properties.setProperty("server." + thisSpec.getServerId(), String.format("%s:%d:%d;%s:%d", thisSpec.getHostname(), thisSpec.getQuorumPort(), thisSpec.getElectionPort(), thisSpec.getHostname(), thisSpec.getPort()));
+                int clientPort = thisSpec == spec ? instancePort : thisSpec.getPort();
+                properties.setProperty("server." + thisSpec.getServerId(), String.format("%s:%d:%d;%s:%d", thisSpec.getHostname(), thisSpec.getQuorumPort(), thisSpec.getElectionPort(), thisSpec.getHostname(), clientPort));
             }
         }
         Map<String,Object> customProperties = spec.getCustomProperties();
@@ -152,4 +165,8 @@ public class QuorumConfigBuilder implements Closeable
 
         return properties;
     }
+
+    public QuorumPeerConfigBuilder bindInstance(int instanceIndex, int instancePort) {
+        return new QuorumPeerConfigBuilder(this, instanceIndex, instancePort);
+    }
 }
diff --git a/curator-test/src/main/java/org/apache/curator/test/QuorumPeerConfigBuilder.java b/curator-test/src/main/java/org/apache/curator/test/QuorumPeerConfigBuilder.java
new file mode 100644
index 00000000..aef3b83a
--- /dev/null
+++ b/curator-test/src/main/java/org/apache/curator/test/QuorumPeerConfigBuilder.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.curator.test;
+
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+
+import java.util.Properties;
+
+public class QuorumPeerConfigBuilder {
+    private final QuorumConfigBuilder configBuilder;
+    private final int instanceIndex;
+    private final int instancePort;
+
+    QuorumPeerConfigBuilder(QuorumConfigBuilder configBuilder, int instanceIndex, int instancePort) {
+        this.configBuilder = configBuilder;
+        this.instanceIndex = instanceIndex;
+        this.instancePort = instancePort;
+    }
+
+    public boolean isFromRandom() {
+        return configBuilder.isFromRandom();
+    }
+
+    public InstanceSpec getInstanceSpec() {
+        return configBuilder.getInstanceSpec(instanceIndex);
+    }
+
+    public QuorumPeerConfig buildConfig() throws Exception {
+        return configBuilder.buildConfig(instanceIndex, instancePort);
+    }
+
+    public Properties buildProperties() throws Exception {
+        return configBuilder.buildConfigProperties(instanceIndex, instancePort);
+    }
+}
diff --git a/curator-test/src/main/java/org/apache/curator/test/TestingQuorumPeerMain.java b/curator-test/src/main/java/org/apache/curator/test/TestingQuorumPeerMain.java
index a54e8cb3..d237f356 100644
--- a/curator-test/src/main/java/org/apache/curator/test/TestingQuorumPeerMain.java
+++ b/curator-test/src/main/java/org/apache/curator/test/TestingQuorumPeerMain.java
@@ -22,7 +22,6 @@ import java.lang.reflect.Field;
 import java.nio.channels.ServerSocketChannel;
 import org.apache.zookeeper.server.ServerCnxnFactory;
 import org.apache.zookeeper.server.quorum.QuorumPeer;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeerMain;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,9 +31,6 @@ class TestingQuorumPeerMain extends QuorumPeerMain implements ZooKeeperMainFace
     private static final Logger log = LoggerFactory.getLogger(TestingQuorumPeerMain.class);
     private volatile boolean isClosed = false;
 
-    private volatile QuorumConfigBuilder configBuilder;
-    private volatile int instanceIndex;
-
     @Override
     public void kill()
     {
@@ -92,30 +88,20 @@ class TestingQuorumPeerMain extends QuorumPeerMain implements ZooKeeperMainFace
     }
 
     @Override
-    public void configure(QuorumConfigBuilder configBuilder, int instanceIndex) {
-        this.configBuilder = configBuilder;
-        this.instanceIndex = instanceIndex;
-    }
-
-    @Override
-    public QuorumPeerConfig getConfig() throws Exception {
-        if (configBuilder != null) {
-            return configBuilder.buildConfig(instanceIndex);
-        }
-
-        return null;
-    }
-
-    @Override
-    public void start() {
+    public void start(QuorumPeerConfigBuilder configBuilder) {
         new Thread(() -> {
             try {
-                runFromConfig(getConfig());
+                runFromConfig(configBuilder.buildConfig());
             } catch (Exception e) {
-                log.error("From testing server (random state: {}) for instance: {}", configBuilder.isFromRandom(), configBuilder.getInstanceSpec(instanceIndex), e);
+                log.error("From testing server (random state: {}) for instance: {}", configBuilder.isFromRandom(), configBuilder.getInstanceSpec(), e);
             }
         }).start();
 
         blockUntilStarted();
     }
+
+    @Override
+    public int getClientPort() {
+        return quorumPeer == null ? -1 : quorumPeer.getClientPort();
+    }
 }
diff --git a/curator-test/src/main/java/org/apache/curator/test/TestingServer.java b/curator-test/src/main/java/org/apache/curator/test/TestingServer.java
index 9d242435..4a50afcf 100644
--- a/curator-test/src/main/java/org/apache/curator/test/TestingServer.java
+++ b/curator-test/src/main/java/org/apache/curator/test/TestingServer.java
@@ -101,7 +101,7 @@ public class TestingServer implements Closeable
      */
     public TestingServer(int port, File tempDirectory, boolean start) throws Exception
     {
-        this(new InstanceSpec(tempDirectory, port, -1, -1, true, -1), start);
+        this(new InstanceSpec(tempDirectory, Math.max(0, port), -1, -1, true, -1), start);
     }
 
     /**
@@ -123,13 +123,18 @@ public class TestingServer implements Closeable
     }
 
     /**
-     * Return the port being used
+     * Return the port being used or will be used.
      *
      * @return port
+     * @throws IllegalStateException if server is configured to bind to port 0 but not started
      */
     public int getPort()
     {
-        return spec.getPort();
+        int port = spec.getPort();
+        if (port > 0) {
+            return port;
+        }
+        return testingZooKeeperServer.getLocalPort();
     }
 
     /**
@@ -186,9 +191,9 @@ public class TestingServer implements Closeable
      * Returns the connection string to use
      *
      * @return connection string
+     * @throws IllegalStateException if server is configured to bind to port 0 but not started
      */
-    public String getConnectString()
-    {
-        return spec.getConnectString();
+    public String getConnectString() {
+        return spec.getHostname() + ":" + getPort();
     }
 }
\ No newline at end of file
diff --git a/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperMain.java b/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperMain.java
index 646cdc4f..aa702e0c 100644
--- a/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperMain.java
+++ b/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperMain.java
@@ -51,8 +51,6 @@ public class TestingZooKeeperMain implements ZooKeeperMainFace
     private volatile ServerCnxnFactory cnxnFactory;
     private volatile TestZooKeeperServer zkServer;
     private volatile ContainerManager containerManager;
-    private volatile QuorumConfigBuilder configBuilder;
-    private volatile int instanceIndex;
 
     private static final Timing timing = new Timing();
 
@@ -98,13 +96,8 @@ public class TestingZooKeeperMain implements ZooKeeperMainFace
         }
     }
 
-    @Override
-    public QuorumPeerConfig getConfig() throws Exception {
-        if (configBuilder != null) {
-            return configBuilder.buildConfig(instanceIndex);
-        }
-
-        return null;
+    TestZooKeeperServer getZkServer() {
+        return zkServer;
     }
 
     private void runFromConfig(QuorumPeerConfig config) throws Exception
@@ -271,27 +264,26 @@ public class TestingZooKeeperMain implements ZooKeeperMainFace
     }
 
     @Override
-    public void configure(QuorumConfigBuilder configBuilder, int instanceIndex) {
-        this.configBuilder = configBuilder;
-        this.instanceIndex = instanceIndex;
-    }
-
-    @Override
-    public void start() {
+    public void start(QuorumPeerConfigBuilder configBuilder) {
         new Thread(() -> {
             try
             {
-                runFromConfig(getConfig());
+                runFromConfig(configBuilder.buildConfig());
             }
             catch ( Exception e )
             {
-                log.error(String.format("From testing server (random state: %s) for instance: %s", String.valueOf(configBuilder.isFromRandom()), configBuilder.getInstanceSpec(instanceIndex)), e);
+                log.error(String.format("From testing server (random state: %s) for instance: %s", configBuilder.isFromRandom(), configBuilder.getInstanceSpec()), e);
             }
         }, "zk-main-thread").start();
 
         blockUntilStarted();
     }
 
+    @Override
+    public int getClientPort() {
+        return cnxnFactory == null ? -1 : cnxnFactory.getLocalPort();
+    }
+
     public static class TestZooKeeperServer extends ZooKeeperServer
     {
         private final FileTxnSnapLog txnLog;
diff --git a/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperServer.java b/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperServer.java
index 42d94fd4..f12c117c 100644
--- a/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperServer.java
+++ b/curator-test/src/main/java/org/apache/curator/test/TestingZooKeeperServer.java
@@ -23,6 +23,8 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,11 +34,12 @@ import org.slf4j.LoggerFactory;
 public class TestingZooKeeperServer implements Closeable
 {
     private static final Logger log = LoggerFactory.getLogger(TestingZooKeeperServer.class);
-    private static final boolean hasZooKeeperServerEmbedded;
+    static boolean hasZooKeeperServerEmbedded;
 
     private final AtomicReference<State> state = new AtomicReference<>(State.LATENT);
     private final QuorumConfigBuilder configBuilder;
     private final int thisInstanceIndex;
+    private int thisInstancePort;
     private volatile ZooKeeperMainFace main;
 
     static {
@@ -71,6 +74,7 @@ public class TestingZooKeeperServer implements Closeable
 
         this.configBuilder = configBuilder;
         this.thisInstanceIndex = thisInstanceIndex;
+        this.thisInstancePort = configBuilder.getInstanceSpec(thisInstanceIndex).getPort();
         main = createServerMain();
     }
 
@@ -163,7 +167,14 @@ public class TestingZooKeeperServer implements Closeable
             return;
         }
 
-        main.configure(configBuilder, thisInstanceIndex);
-        main.start();
+        main.start(configBuilder.bindInstance(thisInstanceIndex, thisInstancePort));
+        thisInstancePort = main.getClientPort();
+    }
+
+    public int getLocalPort() {
+        if (thisInstancePort == 0) {
+            throw new IllegalStateException("server is configured to bind to port 0 but not started");
+        }
+        return thisInstancePort;
     }
 }
\ No newline at end of file
diff --git a/curator-test/src/main/java/org/apache/curator/test/ZooKeeperMainFace.java b/curator-test/src/main/java/org/apache/curator/test/ZooKeeperMainFace.java
index 5b8521fe..6f812079 100644
--- a/curator-test/src/main/java/org/apache/curator/test/ZooKeeperMainFace.java
+++ b/curator-test/src/main/java/org/apache/curator/test/ZooKeeperMainFace.java
@@ -19,15 +19,12 @@
 package org.apache.curator.test;
 
 import java.io.Closeable;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 
-public interface ZooKeeperMainFace extends Closeable
+interface ZooKeeperMainFace extends Closeable
 {
-    void configure(QuorumConfigBuilder config, int instanceIndex) throws Exception;
-
-    void start();
+    void start(QuorumPeerConfigBuilder configBuilder);
 
     void kill();
 
-    QuorumPeerConfig getConfig() throws Exception;
+    int getClientPort() throws Exception;
 }
diff --git a/curator-test/src/main/java/org/apache/curator/test/ZooKeeperServerEmbeddedAdapter.java b/curator-test/src/main/java/org/apache/curator/test/ZooKeeperServerEmbeddedAdapter.java
index 0e8aeb07..ba7591bc 100644
--- a/curator-test/src/main/java/org/apache/curator/test/ZooKeeperServerEmbeddedAdapter.java
+++ b/curator-test/src/main/java/org/apache/curator/test/ZooKeeperServerEmbeddedAdapter.java
@@ -19,12 +19,20 @@
 
 package org.apache.curator.test;
 
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Properties;
+
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.embedded.ZooKeeperServerEmbedded;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import org.apache.zookeeper.server.util.ConfigUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,47 +41,97 @@ public class ZooKeeperServerEmbeddedAdapter implements ZooKeeperMainFace {
     private static final Duration DEFAULT_STARTUP_TIMEOUT = Duration.ofMinutes(1);
 
     private volatile ZooKeeperServerEmbedded zooKeeperEmbedded;
-    private volatile QuorumConfigBuilder configBuilder;
-    private volatile int instanceIndex;
 
     @Override
-    public void configure(QuorumConfigBuilder config, int instanceIndex) throws Exception {
-        this.configBuilder = config;
-        this.instanceIndex = instanceIndex;
+    public void start(QuorumPeerConfigBuilder configBuilder) {
+        try {
+            final Properties properties = configBuilder.buildProperties();
+            properties.put("admin.enableServer", "false");
 
-        final Properties properties = config.buildConfigProperties(instanceIndex);
-        properties.put("admin.enableServer", "false");
+            final Path dataDir = Paths.get(properties.getProperty("dataDir"));
+            zooKeeperEmbedded = ZooKeeperServerEmbedded.builder()
+                    .configuration(properties)
+                    .baseDir(dataDir.getParent())
+                    .build();
+            log.info("Configure ZooKeeperServerEmbeddedAdapter with properties: {}", properties);
 
-        final Path dataDir = Paths.get(properties.getProperty("dataDir"));
-        zooKeeperEmbedded = ZooKeeperServerEmbedded.builder()
-                .configuration(properties)
-                .baseDir(dataDir.getParent())
-                .build();
-        log.info("Configure ZooKeeperServerEmbeddedAdapter with properties: {}", properties);
+            // Before ZOOKEEPER-4303, there are issues when setting "clientPort" to 0:
+            // * It does not set "clientPortAddress" which causes ZooKeeper started with no
+            //   server cnxn factory to serve client requests.
+            // * It uses "clientPortAddress" to construct connection string but not bound port.
+            //
+            // So here, we hijack start process to circumvent these if there is no fix applied.
+            // * Setup "clientPortAddress" if it is null.
+            // * Setup "clientPortAddress" with bound port after started if above step applied.
+            if (hijackClientPort(0)) {
+                zooKeeperEmbedded.start(DEFAULT_STARTUP_TIMEOUT.toMillis());
+                int port = getServerCnxnFactory().getLocalPort();
+                hijackClientPort(port);
+            } else {
+                zooKeeperEmbedded.start(DEFAULT_STARTUP_TIMEOUT.toMillis());
+            }
+        } catch (Exception e) {
+            throw new FailedServerStartException(e);
+        }
     }
 
     @Override
-    public QuorumPeerConfig getConfig() throws Exception {
-        if (configBuilder != null) {
-            return configBuilder.buildConfig(instanceIndex);
+    public int getClientPort() throws Exception {
+        String address = zooKeeperEmbedded.getConnectionString();
+        try {
+            String[] parts = ConfigUtils.getHostAndPort(address);
+            return Integer.parseInt(parts[1], 10);
+        } catch (Exception ex) {
+            throw new IllegalStateException("invalid connection string: " + address);
         }
-
-        return null;
     }
 
-    @Override
-    public void start() {
-        if (zooKeeperEmbedded == null) {
-            throw new FailedServerStartException(new NullPointerException("zooKeeperEmbedded"));
+    private boolean hijackClientPort(int port) {
+        try {
+            Class<?> clazz = Class.forName("org.apache.zookeeper.server.embedded.ZooKeeperServerEmbeddedImpl");
+            Field configField = clazz.getDeclaredField("config");
+            configField.setAccessible(true);
+            QuorumPeerConfig peerConfig = (QuorumPeerConfig) configField.get(zooKeeperEmbedded);
+            if (peerConfig.getClientPortAddress() == null || port != 0) {
+                Field addressField = QuorumPeerConfig.class.getDeclaredField("clientPortAddress");
+                addressField.setAccessible(true);
+                addressField.set(peerConfig, new InetSocketAddress(port));
+                return true;
+            }
+        } catch (Exception ignored) {
+            // swallow hijack failure to accommodate possible upstream changes
         }
+        return false;
+    }
 
+    public ServerCnxnFactory getServerCnxnFactory() {
         try {
-            zooKeeperEmbedded.start(DEFAULT_STARTUP_TIMEOUT.toMillis());
-        } catch (Exception e) {
-            throw new FailedServerStartException(e);
+            Class<?> clazz = Class.forName("org.apache.zookeeper.server.embedded.ZooKeeperServerEmbeddedImpl");
+            Field clusterField = clazz.getDeclaredField("maincluster");
+            clusterField.setAccessible(true);
+            QuorumPeerMain quorumPeerMain = (QuorumPeerMain) clusterField.get(zooKeeperEmbedded);
+            if (quorumPeerMain != null) {
+                Field quorumPeerField = QuorumPeerMain.class.getDeclaredField("quorumPeer");
+                quorumPeerField.setAccessible(true);
+                QuorumPeer quorumPeer = (QuorumPeer) quorumPeerField.get(quorumPeerMain);
+                return getServerCnxnFactory(QuorumPeer.class, quorumPeer, "cnxnFactory");
+            }
+            Field serverField = clazz.getDeclaredField("mainsingle");
+            serverField.setAccessible(true);
+            ZooKeeperServerMain server = (ZooKeeperServerMain) serverField.get(zooKeeperEmbedded);
+            return getServerCnxnFactory(ZooKeeperServerMain.class, server, "cnxnFactory");
+        } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException ex) {
+            throw new IllegalStateException("zk server cnxn factory not found", ex);
         }
     }
 
+    static ServerCnxnFactory getServerCnxnFactory(Class<?> clazz, Object obj, String fieldName)
+            throws NoSuchFieldException, IllegalAccessException {
+        Field cnxnFactoryField = clazz.getDeclaredField(fieldName);
+        cnxnFactoryField.setAccessible(true);
+        return (ServerCnxnFactory) cnxnFactoryField.get(obj);
+    }
+
     @Override
     public void kill() {
         close();
diff --git a/curator-test/src/test/java/org/apache/curator/test/TestTestingServer.java b/curator-test/src/test/java/org/apache/curator/test/TestTestingServer.java
index f16b2ac7..09ac4470 100644
--- a/curator-test/src/test/java/org/apache/curator/test/TestTestingServer.java
+++ b/curator-test/src/test/java/org/apache/curator/test/TestTestingServer.java
@@ -35,6 +35,8 @@ public class TestTestingServer {
 
    @Test
    public void setCustomTickTimeTest() throws Exception {
+      TestingZooKeeperServer.hasZooKeeperServerEmbedded = false;
+
       final int defaultZkTickTime = ZooKeeperServer.DEFAULT_TICK_TIME;
       final int customTickMs;
       if (defaultZkTickTime > 0) {
@@ -45,7 +47,8 @@ public class TestTestingServer {
       final InstanceSpec spec = new InstanceSpec(zkTmpDir, -1, -1, -1, true, -1, customTickMs, -1);
       final int zkTickTime;
       try (TestingServer testingServer = new TestingServer(spec, true)) {
-         zkTickTime = testingServer.getTestingZooKeeperServer().getMain().getConfig().getTickTime();
+         TestingZooKeeperMain main = (TestingZooKeeperMain) testingServer.getTestingZooKeeperServer().getMain();
+         zkTickTime = main.getZkServer().getTickTime();
       }
       assertEquals(customTickMs, zkTickTime);
    }