You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by "siyao.apache.org via common-commits" <co...@hadoop.apache.org> on 2026/10/31 19:52:10 UTC
(hadoop) 03/06: HADOOP-19962. Support IPv6 CIDR rules in MachineList (#8812)
This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e3cdbad7de6f0199b9b4fff058af414c9f2a1371
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Tue Oct 13 01:06:35 2026 -0700
HADOOP-19962. Support IPv6 CIDR rules in MachineList (#8812)
(cherry picked from commit 90a3271609155685a1d37a2efbfbf605d3a9af2e)
---
.../java/org/apache/hadoop/util/MachineList.java | 69 +++++++++-
.../authorize/TestServiceAuthorization.java | 35 +++++
.../org/apache/hadoop/util/TestMachineList.java | 153 +++++++++++++++++++++
3 files changed, 252 insertions(+), 5 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index 68c5c3ca47f..e73fec2fa06 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.util;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -30,6 +32,7 @@
import org.apache.commons.net.util.SubnetUtils;
import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +41,7 @@
* answers membership queries.
*
* Accepts list of ip addresses, ip addreses in CIDR format and/or
- * host addresses.
+ * host addresses. A CIDR entry is either an IPv4 or an IPv6 address range.
*/
public class MachineList {
@@ -60,10 +63,49 @@ public InetAddress getByName (String host) throws UnknownHostException {
}
}
+ /**
+ * An IPv6 address range in CIDR format, which SubnetUtils does not support.
+ */
+ private static final class Inet6Subnet {
+ private static final int MAX_PREFIX_LENGTH = 128;
+
+ private final byte[] network;
+ private final int prefixLength;
+
+ Inet6Subnet(String cidr) {
+ // Any other character is rejected, in particular a scope id because the prefix match
+ // would silently ignore it.
+ if (!cidr.matches("[0-9A-Fa-f:.]+/\\d{1,3}")) {
+ throw new IllegalArgumentException("Could not parse [" + cidr + "]");
+ }
+ int slash = cidr.indexOf('/');
+ InetAddress address = InetAddresses.forString(cidr.substring(0, slash));
+ prefixLength = Integer.parseInt(cidr.substring(slash + 1));
+ // An IPv4-mapped address is parsed as an Inet4Address and belongs in an IPv4 CIDR.
+ if (!(address instanceof Inet6Address) || prefixLength > MAX_PREFIX_LENGTH) {
+ throw new IllegalArgumentException("Could not parse [" + cidr + "]");
+ }
+ network = address.getAddress();
+ }
+
+ boolean isInRange(byte[] address) {
+ int fullBytes = prefixLength / Byte.SIZE;
+ for (int i = 0; i < fullBytes; i++) {
+ if (address[i] != network[i]) {
+ return false;
+ }
+ }
+ int hostBits = Byte.SIZE - prefixLength % Byte.SIZE;
+ return hostBits == Byte.SIZE
+ || ((address[fullBytes] ^ network[fullBytes]) & 0xFF) >>> hostBits == 0;
+ }
+ }
+
private final boolean all;
private final Set<InetAddress> inetAddresses;
private final Collection<String> entries;
private final List<SubnetUtils.SubnetInfo> cidrAddresses;
+ private final List<Inet6Subnet> inet6CidrAddresses;
private final InetAddressFactory addressFactory;
/**
@@ -101,17 +143,23 @@ public MachineList(Collection<String> hostEntries,
all = true;
inetAddresses = null;
cidrAddresses = null;
+ inet6CidrAddresses = null;
} else {
all = false;
Set<InetAddress> addrs = new HashSet<>();
List<SubnetUtils.SubnetInfo> cidrs = new LinkedList<SubnetUtils.SubnetInfo>();
+ List<Inet6Subnet> inet6Cidrs = new ArrayList<>();
for (String hostEntry : hostEntries) {
//ip address range
if (hostEntry.indexOf("/") > -1) {
try {
- SubnetUtils subnet = new SubnetUtils(hostEntry);
- subnet.setInclusiveHostCount(true);
- cidrs.add(subnet.getInfo());
+ if (hostEntry.indexOf(':') > -1) {
+ inet6Cidrs.add(new Inet6Subnet(hostEntry));
+ } else {
+ SubnetUtils subnet = new SubnetUtils(hostEntry);
+ subnet.setInclusiveHostCount(true);
+ cidrs.add(subnet.getInfo());
+ }
} catch (IllegalArgumentException e) {
LOG.warn("Invalid CIDR syntax : " + hostEntry);
throw e;
@@ -126,11 +174,13 @@ public MachineList(Collection<String> hostEntries,
}
inetAddresses = (addrs.size() > 0) ? addrs : null;
cidrAddresses = (cidrs.size() > 0) ? cidrs : null;
+ inet6CidrAddresses = (inet6Cidrs.size() > 0) ? inet6Cidrs : null;
}
} else {
all = false;
inetAddresses = null;
cidrAddresses = null;
+ inet6CidrAddresses = null;
entries = Collections.emptyList();
}
}
@@ -175,7 +225,8 @@ public boolean includes(InetAddress address) {
return true;
}
// iterate through the ip ranges for inclusion
- if (cidrAddresses != null) {
+ // an address never matches an ip range of the other address family
+ if (cidrAddresses != null && address instanceof Inet4Address) {
String ipAddress = address.getHostAddress();
for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) {
if(cidrAddress.isInRange(ipAddress)) {
@@ -183,6 +234,14 @@ public boolean includes(InetAddress address) {
}
}
}
+ if (inet6CidrAddresses != null && address instanceof Inet6Address) {
+ byte[] ipAddress = address.getAddress();
+ for (Inet6Subnet cidrAddress : inet6CidrAddresses) {
+ if (cidrAddress.isInRange(ipAddress)) {
+ return true;
+ }
+ }
+ }
return false;
}
/**
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
index c14972056cf..16646e17476 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.security.authorize;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
@@ -338,6 +340,39 @@ public void testBlockedMachineList() throws UnknownHostException {
}
}
+ @Test
+ public void testBlockedMachineListWithIPv6Client() throws UnknownHostException {
+ UserGroupInformation drwho =
+ UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+ new String[] {"group1", "group2"});
+ InetAddress ipv6Client = InetAddress.getByName("2001:db8::1");
+
+ ServiceAuthorizationManager serviceAuthorizationManager =
+ new ServiceAuthorizationManager();
+ Configuration conf = new Configuration();
+
+ // a blocked MachineList with only IPv4 ranges does not block an IPv6 client
+ conf.set(BLOCKED_HOST_CONFIG, IP_RANGE);
+ serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+ assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
+ TestProtocol.class, conf, ipv6Client)).doesNotThrowAnyException();
+
+ // a blocked MachineList with a matching IPv6 range blocks the IPv6 client
+ conf.set(BLOCKED_HOST_CONFIG, IP_RANGE + ",2001:db8::/32");
+ serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+ assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
+ TestProtocol.class, conf, ipv6Client))
+ .isInstanceOf(AuthorizationException.class)
+ .hasMessageContaining("Host 2001:db8:0:0:0:0:0:1 is not authorized");
+ assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
+ TestProtocol.class, conf, InetAddress.getByName("10.222.0.0")))
+ .isInstanceOf(AuthorizationException.class);
+ // an IPv6 client outside of the blocked range is not blocked
+ assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
+ TestProtocol.class, conf, InetAddress.getByName("2001:db9::1")))
+ .doesNotThrowAnyException();
+ }
+
@Test
public void testDefaultBlockedMachineList() throws UnknownHostException {
UserGroupInformation drwho =
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
index f5c38937895..90d2e64d610 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
@@ -17,12 +17,15 @@
*/
package org.apache.hadoop.util;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
+import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
@@ -42,6 +45,9 @@ public class TestMachineList {
private static String INVALID_CIDR = "10.241/24";
private static String IP_CIDR_LIST =
"10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24";
+ private static final String IPV6_CIDR_LIST = "2001:db8::/32,fd00:1:2:3::/64";
+ private static final String IPV4_IPV6_LIST =
+ "10.222.0.0/16,2001:db8::/32,10.119.103.110,fd00::5";
private static String HOST_LIST = "host1,host4";
private static String HOSTNAME_IP_CIDR_LIST =
"host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,";
@@ -310,4 +316,151 @@ public void testGetCollection() {
assertTrue(col.contains(item));
}
}
+
+ @Test
+ public void testIPv4CIDREdgeCases() {
+ MachineList ml = new MachineList("10.1.2.3/32,10.1.3.2/31", new TestAddressFactory());
+
+ assertThat(ml.includes("10.1.2.2")).isFalse();
+ assertThat(ml.includes("10.1.2.3")).isTrue();
+ assertThat(ml.includes("10.1.2.4")).isFalse();
+
+ assertThat(ml.includes("10.1.3.1")).isFalse();
+ assertThat(ml.includes("10.1.3.2")).isTrue();
+ assertThat(ml.includes("10.1.3.3")).isTrue();
+ assertThat(ml.includes("10.1.3.4")).isFalse();
+
+ ml = new MachineList("0.0.0.0/0", new TestAddressFactory());
+ assertThat(ml.includes("1.2.3.4")).isTrue();
+ assertThat(ml.includes("255.255.255.255")).isTrue();
+ assertThat(ml.includes("2001:db8::1")).isFalse();
+
+ assertThatThrownBy(() -> new MachineList("10.0.0.0/33", new TestAddressFactory()))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void testIPv6List() {
+ MachineList ml = new MachineList("2001:db8::1,fd00::5", new TestAddressFactory());
+
+ assertThat(ml.includes("2001:db8::1")).isTrue();
+ assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
+ assertThat(ml.includes("fd00::5")).isTrue();
+ assertThat(ml.includes("2001:db8::2")).isFalse();
+ assertThat(ml.includes("10.119.103.112")).isFalse();
+ }
+
+ @Test
+ public void testIPv6CIDRs() {
+ MachineList ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());
+
+ assertThat(ml.includes("2001:db7:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse();
+ assertThat(ml.includes("2001:db8::")).isTrue();
+ assertThat(ml.includes("2001:db8::1")).isTrue();
+ assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
+ assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+ assertThat(ml.includes("2001:db9::")).isFalse();
+
+ assertThat(ml.includes("fd00:1:2:2:ffff:ffff:ffff:ffff")).isFalse();
+ assertThat(ml.includes("fd00:1:2:3::")).isTrue();
+ assertThat(ml.includes("fd00:1:2:3:ffff:ffff:ffff:ffff")).isTrue();
+ assertThat(ml.includes("fd00:1:2:4::")).isFalse();
+
+ // host bits in the entry are ignored, as for an IPv4 CIDR
+ ml = new MachineList("2001:db8::1/32", new TestAddressFactory());
+ assertThat(ml.includes("2001:db8:ffff::2")).isTrue();
+ assertThat(ml.includes("2001:db9::1")).isFalse();
+ }
+
+ @Test
+ public void testIPv6CIDRPrefixNotOnByteBoundary() {
+ MachineList ml =
+ new MachineList("2001:db8:8000::/33,fd00:0:0:8::/61", new TestAddressFactory());
+
+ assertThat(ml.includes("2001:db8:7fff:ffff:ffff:ffff:ffff:ffff")).isFalse();
+ assertThat(ml.includes("2001:db8:8000::")).isTrue();
+ assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+ assertThat(ml.includes("2001:db9::")).isFalse();
+
+ assertThat(ml.includes("fd00:0:0:7:ffff:ffff:ffff:ffff")).isFalse();
+ assertThat(ml.includes("fd00:0:0:8::")).isTrue();
+ assertThat(ml.includes("fd00:0:0:f:ffff:ffff:ffff:ffff")).isTrue();
+ assertThat(ml.includes("fd00:0:0:10::")).isFalse();
+ }
+
+ @Test
+ public void testIPv6CIDRWithShortestAndLongestPrefix() {
+ MachineList ml = new MachineList("2001:db8::1/128", new TestAddressFactory());
+ assertThat(ml.includes("2001:db8::")).isFalse();
+ assertThat(ml.includes("2001:db8::1")).isTrue();
+ assertThat(ml.includes("2001:db8::2")).isFalse();
+ assertThat(ml.includes("2001:db8::8001")).isFalse();
+
+ ml = new MachineList("::/0", new TestAddressFactory());
+ assertThat(ml.includes("::")).isTrue();
+ assertThat(ml.includes("2001:db8::1")).isTrue();
+ assertThat(ml.includes("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+ assertThat(ml.includes("10.119.103.112")).isFalse();
+ }
+
+ @Test
+ public void testIPv4AndIPv6CIDRs() {
+ MachineList ml = new MachineList(IPV4_IPV6_LIST, new TestAddressFactory());
+
+ assertThat(ml.includes("10.222.255.255")).isTrue();
+ assertThat(ml.includes("10.223.0.0")).isFalse();
+ assertThat(ml.includes("10.119.103.110")).isTrue();
+ assertThat(ml.includes("2001:db8:ffff::1")).isTrue();
+ assertThat(ml.includes("2001:db9::1")).isFalse();
+ assertThat(ml.includes("fd00::5")).isTrue();
+ assertThat(ml.includes("fd00::6")).isFalse();
+ }
+
+ @Test
+ public void testCIDRsOfOtherAddressFamily() throws UnknownHostException {
+ // an IPv6 address is not in an IPv4 range, the lookup must not throw
+ MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
+ assertThat(ml.includes("2001:db8::1")).isFalse();
+ assertThat(ml.includes(InetAddress.getByName("2001:db8::1"))).isFalse();
+ assertThat(ml.includes("::10.222.0.1")).isFalse();
+
+ ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());
+ assertThat(ml.includes("10.222.0.1")).isFalse();
+ // 32.1.13.184 has the same leading bytes as 2001:db8::
+ assertThat(ml.includes(InetAddress.getByName("32.1.13.184"))).isFalse();
+ }
+
+ @Test
+ public void testIPv4MappedIPv6Address() {
+ // the JDK turns an IPv4-mapped IPv6 address into an IPv4 address
+ MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
+ assertThat(ml.includes("::ffff:10.222.0.1")).isTrue();
+ assertThat(ml.includes("::ffff:10.223.0.0")).isFalse();
+
+ ml = new MachineList("::/0", new TestAddressFactory());
+ assertThat(ml.includes("::ffff:10.222.0.1")).isFalse();
+ }
+
+ @Test
+ public void testIPv6CIDRIgnoresScopeIdOfAddress() throws UnknownHostException {
+ MachineList ml = new MachineList("fe80::/10", new TestAddressFactory());
+ byte[] linkLocal = InetAddress.getByName("fe80::1").getAddress();
+ byte[] siteLocal = InetAddress.getByName("fec0::1").getAddress();
+
+ assertThat(ml.includes(Inet6Address.getByAddress(null, linkLocal, 1))).isTrue();
+ assertThat(ml.includes(Inet6Address.getByAddress(null, siteLocal, 1))).isFalse();
+ }
+
+ @Test
+ public void testInvalidIPv6CIDR() {
+ String[] invalidCidrs = {"2001:db8::/129", "2001:db8::/1000", "2001:db8::/-1",
+ "2001:db8::/+32", "2001:db8::/x", "2001:db8::/", "2001:db8::/32/64", "2001:db8::g/32",
+ ":/32", "[2001:db8::]/32", "fe80::%1/64", "fe80::%/64", "2001:db8::\uff11/128",
+ "::ffff:10.222.0.0/112"};
+ for (String invalidCidr : invalidCidrs) {
+ assertThatThrownBy(() -> new MachineList(invalidCidr, new TestAddressFactory()))
+ .as(invalidCidr)
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org