You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/05/07 20:35:22 UTC

[1/2] kudu git commit: [java] Fix TestUtils on Java 9+

Repository: kudu
Updated Branches:
  refs/heads/master 9ad7cf3e5 -> 8cd7c05b6


[java] Fix TestUtils on Java 9+

Java 9 made some improvments to the
process API causing expected classnames to
change and break TestUtils.pidOfProcess.

This change optimistically tries to access the pid field
of any Process passed to TestUtils.pidOfProcess and
throws an exception if the field could not be found
or accessed.

Change-Id: Ied4ffdca10dc5a0cdc0516964bf6202fd3871f30
Reviewed-on: http://gerrit.cloudera.org:8080/10324
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3f07d059
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3f07d059
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3f07d059

Branch: refs/heads/master
Commit: 3f07d059246579950b0584bfaf7394898b8cdf6c
Parents: 9ad7cf3
Author: Grant Henke <gr...@apache.org>
Authored: Mon May 7 09:57:14 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Mon May 7 19:57:27 2018 +0000

----------------------------------------------------------------------
 .../java/org/apache/kudu/client/TestUtils.java     | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/3f07d059/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
index c3a12bd..5ca26a4 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
@@ -40,8 +40,6 @@ import org.apache.kudu.master.Master;
 public class TestUtils {
   private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
 
-  // Used by pidOfProcess()
-  private static String UNIX_PROCESS_CLS_NAME =  "java.lang.UNIXProcess";
   private static Set<String> VALID_SIGNALS =  ImmutableSet.of("STOP", "CONT", "TERM", "KILL");
 
   private static final String BIN_DIR_PROP = "binDir";
@@ -127,13 +125,16 @@ public class TestUtils {
    */
   static int pidOfProcess(Process proc) throws Exception {
     Class<?> procCls = proc.getClass();
-    if (!procCls.getName().equals(UNIX_PROCESS_CLS_NAME)) {
-      throw new IllegalArgumentException("stopProcess() expects objects of class " +
-          UNIX_PROCESS_CLS_NAME + ", but " + procCls.getName() + " was passed in instead!");
+    try {
+      // Note: In Java 9+ you can call proc.getPid(). We use reflection
+      // because we support earlier version of java.
+      Field pidField = procCls.getDeclaredField("pid");
+      pidField.setAccessible(true);
+      return (Integer) pidField.get(proc);
+    } catch (ReflectiveOperationException e) {
+      throw new IllegalArgumentException("pidOfProcess could not find or access the pid field" +
+          "via reflection on the class " + procCls.getName(), e);
     }
-    Field pidField = procCls.getDeclaredField("pid");
-    pidField.setAccessible(true);
-    return (Integer) pidField.get(proc);
   }
 
   /**


[2/2] kudu git commit: KUDU-2403: Fix FakeDNS to work in Java 9+

Posted by gr...@apache.org.
KUDU-2403: Fix FakeDNS to work in Java 9+

This fixes the FakeDNS implementation in Java 9+ by
first trying to use reflection to replace the internal
java.net.InetAddress$NameService implementation (Java 9 or later)
with our own. If the java.net.InetAddress$NameService field
doesn’t exist, we add our NameService
implementation to the java.net.InetAddress$NameServices
field (Java 8 or earlier).

All Java versions have the same NameService interface.
The main difference is the field names and class location.
That allows us to use the same implementation to replace
the default NameService in all versions.

Change-Id: I319f42b6b878531325d6a48ce6ccdcc9b40fe2bc
Reviewed-on: http://gerrit.cloudera.org:8080/10321
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Grant Henke <gr...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8cd7c05b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8cd7c05b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8cd7c05b

Branch: refs/heads/master
Commit: 8cd7c05b62483a7cf9481f5ae117f9d4e79cc61d
Parents: 3f07d05
Author: Grant Henke <gr...@apache.org>
Authored: Sun May 6 13:08:07 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Mon May 7 20:24:48 2018 +0000

----------------------------------------------------------------------
 java/gradle/tests.gradle                        | 18 +++-
 .../java/org/apache/kudu/client/FakeDNS.java    | 97 ++++++++++++++++----
 2 files changed, 98 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/8cd7c05b/java/gradle/tests.gradle
----------------------------------------------------------------------
diff --git a/java/gradle/tests.gradle b/java/gradle/tests.gradle
index 6963d2e..901ea6e 100644
--- a/java/gradle/tests.gradle
+++ b/java/gradle/tests.gradle
@@ -34,7 +34,23 @@ tasks.withType(Test) {
   forkEvery 1
 
   // Enable assertions during tests.
-  jvmArgs "-enableassertions"
+  jvmArgs += "-enableassertions"
+  // Hide the Java 9+ warnings about illegal reflective access
+  if (JavaVersion.current().isJava9Compatible()) {
+    jvmArgs += "--illegal-access=permit"
+    def reflectionModules = [
+        "java.base/java.lang", // java.lang reflection is used by TestUtils
+        "java.base/java.net",  // java.net reflection is used by FakeDNS
+        "java.base/java.nio",  // java.nio reflection is used by Protobuf
+        "java.base/java.util", // java.util reflection is used by Spark
+        "java.base/java.util.concurrent", // java.util.concurrent reflection is used by Spark
+        "java.base/sun.nio.ch", // sun.nio.ch reflection is used by Netty
+        "java.security.jgss/sun.security.krb5" // sun.security.krb5 reflection is used by Hadoop's KerberosUtil
+    ]
+    reflectionModules.forEach { module ->
+      jvmArgs += "--add-opens=$module=ALL-UNNAMED"
+    }
+  }
 
   // Set a few system properties.
   systemProperty "java.awt.headless", true

http://git-wip-us.apache.org/repos/asf/kudu/blob/8cd7c05b/java/kudu-client/src/test/java/org/apache/kudu/client/FakeDNS.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/FakeDNS.java b/java/kudu-client/src/test/java/org/apache/kudu/client/FakeDNS.java
index db9cd04..caff049 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/FakeDNS.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/FakeDNS.java
@@ -18,17 +18,18 @@
 package org.apache.kudu.client;
 
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.concurrent.GuardedBy;
 
 import com.google.common.base.Throwables;
 import com.google.common.net.InetAddresses;
-import sun.net.spi.nameservice.NameService;
 
 /**
  * Fake DNS resolver which allows our tests to work well even though we use
@@ -76,21 +77,59 @@ public class FakeDNS {
   public synchronized void install() {
     if (installed) return;
     try {
-      Field field = InetAddress.class.getDeclaredField("nameServices");
-      field.setAccessible(true);
-      @SuppressWarnings("unchecked")
-      List<NameService> nameServices = (List<NameService>) field.get(null);
-      nameServices.add(0, new NSImpl());
+      try {
+        // Override the NameService in Java 9 or later.
+        final Class<?> nameServiceInterface = Class.forName("java.net.InetAddress$NameService");
+        Field field = InetAddress.class.getDeclaredField("nameService");
+        // Get the default NameService to fallback to.
+        Method method = InetAddress.class.getDeclaredMethod("createNameService");
+        method.setAccessible(true);
+        Object fallbackNameService = method.invoke(null);
+        // Create a proxy instance to set on the InetAddress field which will handle
+        // all NameService calls.
+        Object proxy = Proxy.newProxyInstance(nameServiceInterface.getClassLoader(),
+            new Class<?>[]{nameServiceInterface}, new NameServiceListener(fallbackNameService));
+        field.setAccessible(true);
+        field.set(InetAddress.class, proxy);
+      } catch (final ClassNotFoundException | NoSuchFieldException e) {
+        // Override the NameService in Java 8 or earlier.
+        final Class<?> nameServiceInterface = Class.forName("sun.net.spi.nameservice.NameService");
+        Field field = InetAddress.class.getDeclaredField("nameServices");
+        // Get the default NameService to fallback to.
+        Method method = InetAddress.class.getDeclaredMethod("createNSProvider", String.class);
+        method.setAccessible(true);
+        Object fallbackNameService = method.invoke(null, "default");
+        // Create a proxy instance to set on the InetAddress field which will handle
+        // all NameService calls.
+        Object proxy = Proxy.newProxyInstance(nameServiceInterface.getClassLoader(),
+            new Class<?>[]{nameServiceInterface}, new NameServiceListener(fallbackNameService));
+        field.setAccessible(true);
+        // Java 8 or earlier takes a list of NameServices
+        field.set(InetAddress.class, Arrays.asList(proxy));
+      }
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
     installed = true;
   }
 
-  private class NSImpl implements NameService {
-    @Override
-    public InetAddress[] lookupAllHostAddr(String host)
-        throws UnknownHostException {
+  /**
+   * The NameService in all versions of Java has the same interface, so we
+   * can use the same InvocationHandler as our proxy instance for both
+   * java.net.InetAddress$NameService and sun.net.spi.nameservice.NameService.
+   */
+  private class NameServiceListener implements InvocationHandler {
+
+    private final Object fallbackNameService;
+
+    // Creates a NameServiceListener with a NameService implementation to
+    // fallback to. The parameter is untyped so we can handle the NameService
+    // type in all versions of Java with reflection.
+    NameServiceListener(Object fallbackNameService) {
+      this.fallbackNameService = fallbackNameService;
+    }
+
+    private InetAddress[] lookupAllHostAddr(String host) throws UnknownHostException {
       InetAddress inetAddress;
       synchronized(FakeDNS.this) {
         inetAddress = forwardResolutions.get(host);
@@ -99,12 +138,18 @@ public class FakeDNS {
         return new InetAddress[]{inetAddress};
       }
 
-      // JDK chains to the next provider automatically.
-      throw new UnknownHostException();
+      try {
+        Method method = fallbackNameService.getClass()
+            .getDeclaredMethod("lookupAllHostAddr", String.class);
+        method.setAccessible(true);
+        return (InetAddress[]) method.invoke(fallbackNameService, host);
+      } catch (ReflectiveOperationException e) {
+        Throwables.propagateIfPossible(e.getCause(), UnknownHostException.class);
+        throw new AssertionError("unexpected reflection issue", e);
+      }
     }
 
-    @Override
-    public String getHostByAddr(byte[] addr) throws UnknownHostException {
+    private String getHostByAddr(byte[] addr) throws UnknownHostException {
       if (addr[0] == 127) {
         return InetAddresses.toAddrString(InetAddress.getByAddress(addr));
       }
@@ -117,7 +162,27 @@ public class FakeDNS {
         return hostname;
       }
 
-      throw new UnknownHostException();
+      try {
+        Method method = fallbackNameService.getClass()
+            .getDeclaredMethod("getHostByAddr", byte[].class);
+        method.setAccessible(true);
+        return (String) method.invoke(fallbackNameService, (Object) addr);
+      } catch (ReflectiveOperationException e) {
+        Throwables.propagateIfPossible(e.getCause(), UnknownHostException.class);
+        throw new AssertionError("unexpected reflection issue", e);
+      }
+    }
+
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+      switch (method.getName()) {
+        case "lookupAllHostAddr":
+          return lookupAllHostAddr((String) args[0]);
+        case "getHostByAddr":
+          return getHostByAddr((byte[]) args[0]);
+        default:
+          throw new UnsupportedOperationException();
+      }
     }
   }
 }