[tez] branch master updated: TEZ-4228: TezClassLoader should be used in TezChild and for Configuration objects

[ab...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/master by this push:
     new 79048f9  TEZ-4228: TezClassLoader should be used in TezChild and for Configuration objects
79048f9 is described below

commit 79048f9e44cc41154a3d741090053a0b0e6f21b0
Author: Laszlo Bodor <bo...@gmail.com>
AuthorDate: Tue Sep 1 12:42:10 2020 +0200

    TEZ-4228: TezClassLoader should be used in TezChild and for Configuration objects
---
 .../java/org/apache/tez/common/TezClassLoader.java     | 18 ++++++++++++++++++
 .../src/main/java/org/apache/tez/common/TezUtils.java  |  3 +++
 .../java/org/apache/tez/runtime/task/TezChild.java     |  4 +++-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
index 923d217..53c9e51 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
@@ -18,6 +18,10 @@ import java.net.URLClassLoader;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * ClassLoader to allow addition of new paths to classpath in the runtime.
  *
@@ -30,6 +34,7 @@ import java.security.PrivilegedAction;
  */
 public class TezClassLoader extends URLClassLoader {
   private static final TezClassLoader INSTANCE;
+  private static final Logger LOG = LoggerFactory.getLogger(TezClassLoader.class);
 
   static {
     INSTANCE = AccessController.doPrivileged(new PrivilegedAction<TezClassLoader>() {
@@ -41,6 +46,11 @@ public class TezClassLoader extends URLClassLoader {
 
   private TezClassLoader() {
     super(new URL[] {}, TezClassLoader.class.getClassLoader());
+
+    LOG.info(
+        "Created TezClassLoader with parent classloader: {}, thread: {}, system classloader: {}",
+        TezClassLoader.class.getClassLoader(), Thread.currentThread().getId(),
+        ClassLoader.getSystemClassLoader());
   }
 
   public void addURL(URL url) {
@@ -52,6 +62,14 @@ public class TezClassLoader extends URLClassLoader {
   }
 
   public static void setupTezClassLoader() {
+    LOG.debug(
+        "Setting up TezClassLoader: thread: {}, current thread classloader: {} system classloader: {}",
+        Thread.currentThread().getId(), Thread.currentThread().getContextClassLoader(),
+        ClassLoader.getSystemClassLoader());
     Thread.currentThread().setContextClassLoader(INSTANCE);
   }
+
+  public static void setupForConfiguration(Configuration configuration) {
+    configuration.setClassLoader(INSTANCE);
+  }
 }
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 23811aa..3f02418 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -117,6 +117,7 @@ public class TezUtils {
       DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
       Configuration conf = new Configuration(false);
       readConfFromPB(confProto, conf);
+      TezClassLoader.setupForConfiguration(conf);
       return conf;
     }
   }
@@ -130,6 +131,7 @@ public class TezUtils {
     try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
       DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
       readConfFromPB(confProto, configuration);
+      TezClassLoader.setupForConfiguration(configuration);
       return configuration;
     }
   }
@@ -139,6 +141,7 @@ public class TezUtils {
     try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
       DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
       readConfFromPB(confProto, configuration);
+      TezClassLoader.setupForConfiguration(configuration);
     }
   }
 
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 6dd146a..86ceb12 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -77,6 +77,8 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Function;
 import org.apache.tez.common.Preconditions;
+import org.apache.tez.common.TezClassLoader;
+
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
@@ -480,7 +482,7 @@ public class TezChild {
   }
 
   public static void main(String[] args) throws IOException, InterruptedException, TezException {
-
+    TezClassLoader.setupTezClassLoader();
     final Configuration defaultConf = new Configuration();
 
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());