You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jo...@apache.org on 2023/08/28 21:53:11 UTC
[cassandra] branch cassandra-4.0 updated: Support Dynamic Port Allocation for in-jvm dtest framework
This is an automated email from the ASF dual-hosted git repository.
jonmeredith pushed a commit to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-4.0 by this push:
new 6ffa43f68b Support Dynamic Port Allocation for in-jvm dtest framework
6ffa43f68b is described below
commit 6ffa43f68b8d10ca84d4a00bf81269527b4e14df
Author: Francisco Guerrero <fr...@gmail.com>
AuthorDate: Fri Aug 25 11:10:48 2023 -0600
Support Dynamic Port Allocation for in-jvm dtest framework
patch by Francisco Guerrero; reviewed by Dinesh Joshi, Jon Meredith, Yifan Cai for CASSANDRA-18722
---
.../distributed/impl/AbstractCluster.java | 29 ++++++-
.../distributed/impl/INodeProvisionStrategy.java | 91 ++++++++++++++++++----
.../cassandra/distributed/impl/InstanceConfig.java | 2 +-
.../distributed/test/jmx/JMXFeatureTest.java | 3 +-
.../unit/org/apache/cassandra/net/SocketUtils.java | 57 +++++++++-----
5 files changed, 140 insertions(+), 42 deletions(-)
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index fcbef4c220..f57f2b76e2 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -140,6 +140,7 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
private final Map<Integer, NetworkTopology.DcAndRack> nodeIdTopology;
private final Consumer<IInstanceConfig> configUpdater;
private final int broadcastPort;
+ private final Map<String, Integer> portMap;
// mutated by starting/stopping a node
private final List<I> instances;
@@ -163,6 +164,7 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
extends org.apache.cassandra.distributed.shared.AbstractBuilder<I, C, B>
{
private INodeProvisionStrategy.Strategy nodeProvisionStrategy = INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces;
+ private boolean dynamicPortAllocation = false;
{
// Indicate that we are running in the in-jvm dtest environment
@@ -177,10 +179,32 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
super(factory);
}
+ @SuppressWarnings("unchecked")
+ private B self()
+ {
+ return (B) this;
+ }
+
public B withNodeProvisionStrategy(INodeProvisionStrategy.Strategy nodeProvisionStrategy)
{
this.nodeProvisionStrategy = nodeProvisionStrategy;
- return (B) this;
+ return self();
+ }
+
+ /**
+ * When {@code dynamicPortAllocation} is {@code true}, it will ask {@link INodeProvisionStrategy} to provision
+ * available storage, native and JMX ports in the given interface. When {@code dynamicPortAllocation} is
+ * {@code false} (the default behavior), it will use statically allocated ports based on the number of
+ * interfaces available and the node number.
+ *
+ * @param dynamicPortAllocation {@code true} for dynamic port allocation, {@code false} for static port
+ * allocation
+ * @return a reference to this Builder
+ */
+ public B withDynamicPortAllocation(boolean dynamicPortAllocation)
+ {
+ this.dynamicPortAllocation = dynamicPortAllocation;
+ return self();
}
}
@@ -408,6 +432,7 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
this.filters = new MessageFilters();
this.instanceInitializer = builder.getInstanceInitializer();
this.datadirCount = builder.getDatadirCount();
+ this.portMap = builder.dynamicPortAllocation ? new ConcurrentHashMap<>() : null;
int generation = GENERATION.incrementAndGet();
for (int i = 0; i < builder.getNodeCount(); ++i)
@@ -431,7 +456,7 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
private InstanceConfig createInstanceConfig(int nodeNum)
{
- INodeProvisionStrategy provisionStrategy = nodeProvisionStrategy.create(subnet);
+ INodeProvisionStrategy provisionStrategy = nodeProvisionStrategy.create(subnet, portMap);
long token = tokenSupplier.token(nodeNum);
NetworkTopology topology = buildNetworkTopology(provisionStrategy, nodeIdTopology);
InstanceConfig config = InstanceConfig.generate(nodeNum, provisionStrategy, topology, root, Long.toString(token), datadirCount);
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java b/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java
index 6e8f9ffe26..017b11de26 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java
@@ -18,43 +18,68 @@
package org.apache.cassandra.distributed.impl;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.net.SocketUtils;
+
public interface INodeProvisionStrategy
{
-
- public enum Strategy
+ enum Strategy
{
OneNetworkInterface
{
- INodeProvisionStrategy create(int subnet) {
+ @Override
+ INodeProvisionStrategy create(int subnet, @Nullable Map<String, Integer> portMap)
+ {
+ String ipAdress = "127.0." + subnet + ".1";
return new INodeProvisionStrategy()
{
+ @Override
public String seedIp()
{
- return "127.0." + subnet + ".1";
+ return ipAdress;
}
+ @Override
public int seedPort()
{
- return 7012;
+ return storagePort(1);
}
+ @Override
public String ipAddress(int nodeNum)
{
- return "127.0." + subnet + ".1";
+ return ipAdress;
}
+ @Override
public int storagePort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("storagePort@node" + nodeNum, key -> SocketUtils.findAvailablePort(seedIp(), 7011 + nodeNum));
+ }
return 7011 + nodeNum;
}
+ @Override
public int nativeTransportPort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("nativeTransportPort@node" + nodeNum, key -> SocketUtils.findAvailablePort(seedIp(), 9041 + nodeNum));
+ }
return 9041 + nodeNum;
}
+ @Override
public int jmxPort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("jmxPort@node" + nodeNum, key -> SocketUtils.findAvailablePort(seedIp(), 7199 + nodeNum));
+ }
return 7199 + nodeNum;
}
};
@@ -62,49 +87,81 @@ public interface INodeProvisionStrategy
},
MultipleNetworkInterfaces
{
- INodeProvisionStrategy create(int subnet) {
- String ipPrefix = "127.0." + subnet + ".";
+ @Override
+ INodeProvisionStrategy create(int subnet, @Nullable Map<String, Integer> portMap)
+ {
+ String ipPrefix = "127.0." + subnet + '.';
return new INodeProvisionStrategy()
{
+
+ @Override
public String seedIp()
{
- return ipPrefix + "1";
+ return ipPrefix + '1';
}
+ @Override
public int seedPort()
{
- return 7012;
+ return storagePort(1);
}
+ @Override
public String ipAddress(int nodeNum)
{
return ipPrefix + nodeNum;
}
+ @Override
public int storagePort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("storagePort@node" + nodeNum, key -> SocketUtils.findAvailablePort(ipAddress(nodeNum), 7012));
+ }
return 7012;
}
+ @Override
public int nativeTransportPort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("nativeTransportPort@node" + nodeNum, key -> SocketUtils.findAvailablePort(ipAddress(nodeNum), 9042));
+ }
return 9042;
}
+ @Override
public int jmxPort(int nodeNum)
{
+ if (portMap != null)
+ {
+ return portMap.computeIfAbsent("jmxPort@node" + nodeNum, key -> SocketUtils.findAvailablePort(ipAddress(nodeNum), 7199));
+ }
return 7199;
}
};
}
};
- abstract INodeProvisionStrategy create(int subnet);
+
+ INodeProvisionStrategy create(int subnet)
+ {
+ return create(subnet, null);
+ }
+
+ abstract INodeProvisionStrategy create(int subnet, @Nullable Map<String, Integer> portMap);
}
- abstract String seedIp();
- abstract int seedPort();
- abstract String ipAddress(int nodeNum);
- abstract int storagePort(int nodeNum);
- abstract int nativeTransportPort(int nodeNum);
- abstract int jmxPort(int nodeNum);
+ String seedIp();
+
+ int seedPort();
+
+ String ipAddress(int nodeNum);
+
+ int storagePort(int nodeNum);
+
+ int nativeTransportPort(int nodeNum);
+
+ int jmxPort(int nodeNum);
}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
index 06229a0c01..d1714a33bf 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
@@ -110,7 +110,7 @@ public class InstanceConfig implements IInstanceConfig
.set("native_transport_port", native_transport_port)
.set("endpoint_snitch", DistributedTestSnitch.class.getName())
.set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(),
- Collections.singletonMap("seeds", seedIp + ":" + seedPort)))
+ Collections.singletonMap("seeds", seedIp + ':' + seedPort)))
// required settings for dtest functionality
.set("diagnostic_events_enabled", true)
.set("auto_bootstrap", false)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java
index ea1efa8902..b6b0eba2f6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java
@@ -72,6 +72,7 @@ public class JMXFeatureTest extends TestBaseImpl
for (int i = 0; i < iterations; i++)
{
try (Cluster cluster = Cluster.build(2)
+ .withDynamicPortAllocation(true)
.withNodeProvisionStrategy(provisionStrategy)
.withConfig(c -> c.with(Feature.values())).start())
{
@@ -136,7 +137,7 @@ public class JMXFeatureTest extends TestBaseImpl
// to check that we are actually connecting to the correct instance
String defaultDomain = mbsc.getDefaultDomain();
instancesContacted.add(defaultDomain);
- Assert.assertThat(defaultDomain, startsWith(JMXUtil.getJmxHost(config) + ":" + config.jmxPort()));
+ Assert.assertThat(defaultDomain, startsWith(JMXUtil.getJmxHost(config) + ':' + config.jmxPort()));
}
catch (Throwable t)
{
diff --git a/test/unit/org/apache/cassandra/net/SocketUtils.java b/test/unit/org/apache/cassandra/net/SocketUtils.java
index a0a149029b..6a6cd559e2 100644
--- a/test/unit/org/apache/cassandra/net/SocketUtils.java
+++ b/test/unit/org/apache/cassandra/net/SocketUtils.java
@@ -19,39 +19,54 @@
package org.apache.cassandra.net;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.ServerSocket;
+import java.net.UnknownHostException;
import com.google.common.base.Throwables;
public class SocketUtils
{
- public static synchronized int findAvailablePort() throws RuntimeException
+ /**
+ * Returns an available port for the given {@code bindAddress}. When an {@link IOException} occurs when opening a
+ * socket or if a {@link SecurityException} is raised because a manager exists and its checkListen method does
+ * not allow the operation, the {@code fallbackPort} is returned.
+ *
+ * @param bindAddress the ip address for the interface where we need an available port number
+ * @param fallbackPort a port to return in case {@link SecurityException} or {@link IOException} is encountered
+ * @return an available port the given {@code bindAddress} when succeeds, otherwise the {@code fallbackPort}
+ * @throws RuntimeException if no IP address for the {@code bindAddress} could be found
+ */
+ public static synchronized int findAvailablePort(String bindAddress, int fallbackPort) throws RuntimeException
{
- ServerSocket ss = null;
try
{
- // let the system pick an ephemeral port
- ss = new ServerSocket(0);
- ss.setReuseAddress(true);
- return ss.getLocalPort();
+ return findAvailablePort(InetAddress.getByName(bindAddress), fallbackPort);
}
- catch (IOException e)
+ catch (UnknownHostException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
- finally
+ }
+
+ /**
+ * Returns an available port for the given {@code bindAddress}. When an {@link IOException} occurs when opening a
+ * socket or if a {@link SecurityException} is raised because a manager exists and its checkListen method does
+ * not allow the operation, the {@code fallbackPort} is returned.
+ *
+ * @param bindAddress the ip address for the interface where we need an available port number
+ * @param fallbackPort a port to return in case {@link SecurityException} or {@link IOException} is encountered
+ * @return an available port the given {@code bindAddress} when succeeds, otherwise the {@code fallbackPort}
+ */
+ public static synchronized int findAvailablePort(InetAddress bindAddress, int fallbackPort)
+ {
+ try (ServerSocket socket = new ServerSocket(0, 50, bindAddress))
+ {
+ return socket.getLocalPort();
+ }
+ catch (SecurityException | IOException exception)
{
- if (ss != null)
- {
- try
- {
- ss.close();
- }
- catch (IOException e)
- {
- Throwables.propagate(e);
- }
- }
+ return fallbackPort;
}
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org