You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by "siyao.apache.org via common-commits" <co...@hadoop.apache.org> on 2026/10/31 19:52:10 UTC

(hadoop) 03/06: HADOOP-19962. Support IPv6 CIDR rules in MachineList (#8812)

This is an automated email from the ASF dual-hosted git repository.

smengcl pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit e3cdbad7de6f0199b9b4fff058af414c9f2a1371
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Tue Oct 13 01:06:35 2026 -0700

    HADOOP-19962. Support IPv6 CIDR rules in MachineList (#8812)
    
    (cherry picked from commit 90a3271609155685a1d37a2efbfbf605d3a9af2e)
---
 .../java/org/apache/hadoop/util/MachineList.java   |  69 +++++++++-
 .../authorize/TestServiceAuthorization.java        |  35 +++++
 .../org/apache/hadoop/util/TestMachineList.java    | 153 +++++++++++++++++++++
 3 files changed, 252 insertions(+), 5 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index 68c5c3ca47f..e73fec2fa06 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.util;
 
+import java.net.Inet4Address;
+import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
@@ -30,6 +32,7 @@
 import org.apache.commons.net.util.SubnetUtils;
 
 import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,7 +41,7 @@
  * answers membership queries.
  *
  * Accepts list of ip addresses, ip addreses in CIDR format and/or 
- * host addresses.
+ * host addresses. A CIDR entry is either an IPv4 or an IPv6 address range.
  */
 
 public class MachineList {
@@ -60,10 +63,49 @@ public InetAddress getByName (String host) throws UnknownHostException {
     }
   }
 
+  /**
+   * An IPv6 address range in CIDR format, which SubnetUtils does not support.
+   */
+  private static final class Inet6Subnet {
+    private static final int MAX_PREFIX_LENGTH = 128;
+
+    private final byte[] network;
+    private final int prefixLength;
+
+    Inet6Subnet(String cidr) {
+      // Any other character is rejected, in particular a scope id because the prefix match
+      // would silently ignore it.
+      if (!cidr.matches("[0-9A-Fa-f:.]+/\\d{1,3}")) {
+        throw new IllegalArgumentException("Could not parse [" + cidr + "]");
+      }
+      int slash = cidr.indexOf('/');
+      InetAddress address = InetAddresses.forString(cidr.substring(0, slash));
+      prefixLength = Integer.parseInt(cidr.substring(slash + 1));
+      // An IPv4-mapped address is parsed as an Inet4Address and belongs in an IPv4 CIDR.
+      if (!(address instanceof Inet6Address) || prefixLength > MAX_PREFIX_LENGTH) {
+        throw new IllegalArgumentException("Could not parse [" + cidr + "]");
+      }
+      network = address.getAddress();
+    }
+
+    boolean isInRange(byte[] address) {
+      int fullBytes = prefixLength / Byte.SIZE;
+      for (int i = 0; i < fullBytes; i++) {
+        if (address[i] != network[i]) {
+          return false;
+        }
+      }
+      int hostBits = Byte.SIZE - prefixLength % Byte.SIZE;
+      return hostBits == Byte.SIZE
+          || ((address[fullBytes] ^ network[fullBytes]) & 0xFF) >>> hostBits == 0;
+    }
+  }
+
   private final boolean all;
   private final Set<InetAddress> inetAddresses;
   private final Collection<String> entries;
   private final List<SubnetUtils.SubnetInfo> cidrAddresses;
+  private final List<Inet6Subnet> inet6CidrAddresses;
   private final InetAddressFactory addressFactory;
 
   /**
@@ -101,17 +143,23 @@ public MachineList(Collection<String> hostEntries,
         all = true;
         inetAddresses = null;
         cidrAddresses = null; 
+        inet6CidrAddresses = null;
       } else {
         all = false;
         Set<InetAddress> addrs = new HashSet<>();
         List<SubnetUtils.SubnetInfo> cidrs = new LinkedList<SubnetUtils.SubnetInfo>();
+        List<Inet6Subnet> inet6Cidrs = new ArrayList<>();
         for (String hostEntry : hostEntries) {
           //ip address range
           if (hostEntry.indexOf("/") > -1) {
             try {
-              SubnetUtils subnet = new SubnetUtils(hostEntry);
-              subnet.setInclusiveHostCount(true);
-              cidrs.add(subnet.getInfo());
+              if (hostEntry.indexOf(':') > -1) {
+                inet6Cidrs.add(new Inet6Subnet(hostEntry));
+              } else {
+                SubnetUtils subnet = new SubnetUtils(hostEntry);
+                subnet.setInclusiveHostCount(true);
+                cidrs.add(subnet.getInfo());
+              }
             } catch (IllegalArgumentException e) {
               LOG.warn("Invalid CIDR syntax : " + hostEntry);
               throw e;
@@ -126,11 +174,13 @@ public MachineList(Collection<String> hostEntries,
         }
         inetAddresses = (addrs.size() > 0) ? addrs : null;
         cidrAddresses = (cidrs.size() > 0) ? cidrs : null;
+        inet6CidrAddresses = (inet6Cidrs.size() > 0) ? inet6Cidrs : null;
       }
     } else {
       all = false;
       inetAddresses = null;
       cidrAddresses = null;
+      inet6CidrAddresses = null;
       entries = Collections.emptyList();
     }
   }
@@ -175,7 +225,8 @@ public boolean includes(InetAddress address) {
       return true;
     }
     // iterate through the ip ranges for inclusion
-    if (cidrAddresses != null) {
+    // an address never matches an ip range of the other address family
+    if (cidrAddresses != null && address instanceof Inet4Address) {
       String ipAddress = address.getHostAddress();
       for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) {
         if(cidrAddress.isInRange(ipAddress)) {
@@ -183,6 +234,14 @@ public boolean includes(InetAddress address) {
         }
       }
     }
+    if (inet6CidrAddresses != null && address instanceof Inet6Address) {
+      byte[] ipAddress = address.getAddress();
+      for (Inet6Subnet cidrAddress : inet6CidrAddresses) {
+        if (cidrAddress.isInRange(ipAddress)) {
+          return true;
+        }
+      }
+    }
     return false;
   }
   /**
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
index c14972056cf..16646e17476 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.security.authorize;
 
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -338,6 +340,39 @@ public void testBlockedMachineList() throws UnknownHostException {
     }
   }
 
+  @Test
+  public void testBlockedMachineListWithIPv6Client() throws UnknownHostException {
+    UserGroupInformation drwho =
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+            new String[] {"group1", "group2"});
+    InetAddress ipv6Client = InetAddress.getByName("2001:db8::1");
+
+    ServiceAuthorizationManager serviceAuthorizationManager =
+        new ServiceAuthorizationManager();
+    Configuration conf = new Configuration();
+
+    // a blocked MachineList with only IPv4 ranges does not block an IPv6 client
+    conf.set(BLOCKED_HOST_CONFIG, IP_RANGE);
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
+        TestProtocol.class, conf, ipv6Client)).doesNotThrowAnyException();
+
+    // a blocked MachineList with a matching IPv6 range blocks the IPv6 client
+    conf.set(BLOCKED_HOST_CONFIG, IP_RANGE + ",2001:db8::/32");
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
+        TestProtocol.class, conf, ipv6Client))
+        .isInstanceOf(AuthorizationException.class)
+        .hasMessageContaining("Host 2001:db8:0:0:0:0:0:1 is not authorized");
+    assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
+        TestProtocol.class, conf, InetAddress.getByName("10.222.0.0")))
+        .isInstanceOf(AuthorizationException.class);
+    // an IPv6 client outside of the blocked range is not blocked
+    assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
+        TestProtocol.class, conf, InetAddress.getByName("2001:db9::1")))
+        .doesNotThrowAnyException();
+  }
+
   @Test
   public void testDefaultBlockedMachineList() throws UnknownHostException {
     UserGroupInformation drwho =
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
index f5c38937895..90d2e64d610 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
@@ -17,12 +17,15 @@
  */
 package org.apache.hadoop.util;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
+import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Collection;
@@ -42,6 +45,9 @@ public class TestMachineList {
   private static String INVALID_CIDR = "10.241/24";
   private static String IP_CIDR_LIST =
     "10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24";
+  private static final String IPV6_CIDR_LIST = "2001:db8::/32,fd00:1:2:3::/64";
+  private static final String IPV4_IPV6_LIST =
+      "10.222.0.0/16,2001:db8::/32,10.119.103.110,fd00::5";
   private static String HOST_LIST = "host1,host4";
   private static String HOSTNAME_IP_CIDR_LIST =
     "host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,";
@@ -310,4 +316,151 @@ public void testGetCollection() {
       assertTrue(col.contains(item));
     }
   }
+
+  @Test
+  public void testIPv4CIDREdgeCases() {
+    MachineList ml = new MachineList("10.1.2.3/32,10.1.3.2/31", new TestAddressFactory());
+
+    assertThat(ml.includes("10.1.2.2")).isFalse();
+    assertThat(ml.includes("10.1.2.3")).isTrue();
+    assertThat(ml.includes("10.1.2.4")).isFalse();
+
+    assertThat(ml.includes("10.1.3.1")).isFalse();
+    assertThat(ml.includes("10.1.3.2")).isTrue();
+    assertThat(ml.includes("10.1.3.3")).isTrue();
+    assertThat(ml.includes("10.1.3.4")).isFalse();
+
+    ml = new MachineList("0.0.0.0/0", new TestAddressFactory());
+    assertThat(ml.includes("1.2.3.4")).isTrue();
+    assertThat(ml.includes("255.255.255.255")).isTrue();
+    assertThat(ml.includes("2001:db8::1")).isFalse();
+
+    assertThatThrownBy(() -> new MachineList("10.0.0.0/33", new TestAddressFactory()))
+        .isInstanceOf(IllegalArgumentException.class);
+  }
+
+  @Test
+  public void testIPv6List() {
+    MachineList ml = new MachineList("2001:db8::1,fd00::5", new TestAddressFactory());
+
+    assertThat(ml.includes("2001:db8::1")).isTrue();
+    assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
+    assertThat(ml.includes("fd00::5")).isTrue();
+    assertThat(ml.includes("2001:db8::2")).isFalse();
+    assertThat(ml.includes("10.119.103.112")).isFalse();
+  }
+
+  @Test
+  public void testIPv6CIDRs() {
+    MachineList ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());
+
+    assertThat(ml.includes("2001:db7:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse();
+    assertThat(ml.includes("2001:db8::")).isTrue();
+    assertThat(ml.includes("2001:db8::1")).isTrue();
+    assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
+    assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+    assertThat(ml.includes("2001:db9::")).isFalse();
+
+    assertThat(ml.includes("fd00:1:2:2:ffff:ffff:ffff:ffff")).isFalse();
+    assertThat(ml.includes("fd00:1:2:3::")).isTrue();
+    assertThat(ml.includes("fd00:1:2:3:ffff:ffff:ffff:ffff")).isTrue();
+    assertThat(ml.includes("fd00:1:2:4::")).isFalse();
+
+    // host bits in the entry are ignored, as for an IPv4 CIDR
+    ml = new MachineList("2001:db8::1/32", new TestAddressFactory());
+    assertThat(ml.includes("2001:db8:ffff::2")).isTrue();
+    assertThat(ml.includes("2001:db9::1")).isFalse();
+  }
+
+  @Test
+  public void testIPv6CIDRPrefixNotOnByteBoundary() {
+    MachineList ml =
+        new MachineList("2001:db8:8000::/33,fd00:0:0:8::/61", new TestAddressFactory());
+
+    assertThat(ml.includes("2001:db8:7fff:ffff:ffff:ffff:ffff:ffff")).isFalse();
+    assertThat(ml.includes("2001:db8:8000::")).isTrue();
+    assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+    assertThat(ml.includes("2001:db9::")).isFalse();
+
+    assertThat(ml.includes("fd00:0:0:7:ffff:ffff:ffff:ffff")).isFalse();
+    assertThat(ml.includes("fd00:0:0:8::")).isTrue();
+    assertThat(ml.includes("fd00:0:0:f:ffff:ffff:ffff:ffff")).isTrue();
+    assertThat(ml.includes("fd00:0:0:10::")).isFalse();
+  }
+
+  @Test
+  public void testIPv6CIDRWithShortestAndLongestPrefix() {
+    MachineList ml = new MachineList("2001:db8::1/128", new TestAddressFactory());
+    assertThat(ml.includes("2001:db8::")).isFalse();
+    assertThat(ml.includes("2001:db8::1")).isTrue();
+    assertThat(ml.includes("2001:db8::2")).isFalse();
+    assertThat(ml.includes("2001:db8::8001")).isFalse();
+
+    ml = new MachineList("::/0", new TestAddressFactory());
+    assertThat(ml.includes("::")).isTrue();
+    assertThat(ml.includes("2001:db8::1")).isTrue();
+    assertThat(ml.includes("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
+    assertThat(ml.includes("10.119.103.112")).isFalse();
+  }
+
+  @Test
+  public void testIPv4AndIPv6CIDRs() {
+    MachineList ml = new MachineList(IPV4_IPV6_LIST, new TestAddressFactory());
+
+    assertThat(ml.includes("10.222.255.255")).isTrue();
+    assertThat(ml.includes("10.223.0.0")).isFalse();
+    assertThat(ml.includes("10.119.103.110")).isTrue();
+    assertThat(ml.includes("2001:db8:ffff::1")).isTrue();
+    assertThat(ml.includes("2001:db9::1")).isFalse();
+    assertThat(ml.includes("fd00::5")).isTrue();
+    assertThat(ml.includes("fd00::6")).isFalse();
+  }
+
+  @Test
+  public void testCIDRsOfOtherAddressFamily() throws UnknownHostException {
+    // an IPv6 address is not in an IPv4 range, the lookup must not throw
+    MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
+    assertThat(ml.includes("2001:db8::1")).isFalse();
+    assertThat(ml.includes(InetAddress.getByName("2001:db8::1"))).isFalse();
+    assertThat(ml.includes("::10.222.0.1")).isFalse();
+
+    ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());
+    assertThat(ml.includes("10.222.0.1")).isFalse();
+    // 32.1.13.184 has the same leading bytes as 2001:db8::
+    assertThat(ml.includes(InetAddress.getByName("32.1.13.184"))).isFalse();
+  }
+
+  @Test
+  public void testIPv4MappedIPv6Address() {
+    // the JDK turns an IPv4-mapped IPv6 address into an IPv4 address
+    MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
+    assertThat(ml.includes("::ffff:10.222.0.1")).isTrue();
+    assertThat(ml.includes("::ffff:10.223.0.0")).isFalse();
+
+    ml = new MachineList("::/0", new TestAddressFactory());
+    assertThat(ml.includes("::ffff:10.222.0.1")).isFalse();
+  }
+
+  @Test
+  public void testIPv6CIDRIgnoresScopeIdOfAddress() throws UnknownHostException {
+    MachineList ml = new MachineList("fe80::/10", new TestAddressFactory());
+    byte[] linkLocal = InetAddress.getByName("fe80::1").getAddress();
+    byte[] siteLocal = InetAddress.getByName("fec0::1").getAddress();
+
+    assertThat(ml.includes(Inet6Address.getByAddress(null, linkLocal, 1))).isTrue();
+    assertThat(ml.includes(Inet6Address.getByAddress(null, siteLocal, 1))).isFalse();
+  }
+
+  @Test
+  public void testInvalidIPv6CIDR() {
+    String[] invalidCidrs = {"2001:db8::/129", "2001:db8::/1000", "2001:db8::/-1",
+        "2001:db8::/+32", "2001:db8::/x", "2001:db8::/", "2001:db8::/32/64", "2001:db8::g/32",
+        ":/32", "[2001:db8::]/32", "fe80::%1/64", "fe80::%/64", "2001:db8::\uff11/128",
+        "::ffff:10.222.0.0/112"};
+    for (String invalidCidr : invalidCidrs) {
+      assertThatThrownBy(() -> new MachineList(invalidCidr, new TestAddressFactory()))
+          .as(invalidCidr)
+          .isInstanceOf(IllegalArgumentException.class);
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org