You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2020/09/03 14:12:59 UTC
[tez] branch branch-0.10.0 updated: TEZ-4228: TezClassLoader should
be used in TezChild and for Configuration objects
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch branch-0.10.0
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.10.0 by this push:
new 78d2ed8 TEZ-4228: TezClassLoader should be used in TezChild and for Configuration objects
78d2ed8 is described below
commit 78d2ed822f786d0f23faf22ee44d4e45b69416c1
Author: Laszlo Bodor <bo...@gmail.com>
AuthorDate: Tue Sep 1 12:42:10 2020 +0200
TEZ-4228: TezClassLoader should be used in TezChild and for Configuration objects
---
.../java/org/apache/tez/common/TezClassLoader.java | 18 ++++++++++++++++++
.../src/main/java/org/apache/tez/common/TezUtils.java | 3 +++
.../java/org/apache/tez/runtime/task/TezChild.java | 4 +++-
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
index 923d217..53c9e51 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
@@ -18,6 +18,10 @@ import java.net.URLClassLoader;
import java.security.AccessController;
import java.security.PrivilegedAction;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* ClassLoader to allow addition of new paths to classpath in the runtime.
*
@@ -30,6 +34,7 @@ import java.security.PrivilegedAction;
*/
public class TezClassLoader extends URLClassLoader {
private static final TezClassLoader INSTANCE;
+ private static final Logger LOG = LoggerFactory.getLogger(TezClassLoader.class);
static {
INSTANCE = AccessController.doPrivileged(new PrivilegedAction<TezClassLoader>() {
@@ -41,6 +46,11 @@ public class TezClassLoader extends URLClassLoader {
private TezClassLoader() {
super(new URL[] {}, TezClassLoader.class.getClassLoader());
+
+ LOG.info(
+ "Created TezClassLoader with parent classloader: {}, thread: {}, system classloader: {}",
+ TezClassLoader.class.getClassLoader(), Thread.currentThread().getId(),
+ ClassLoader.getSystemClassLoader());
}
public void addURL(URL url) {
@@ -52,6 +62,14 @@ public class TezClassLoader extends URLClassLoader {
}
public static void setupTezClassLoader() {
+ LOG.debug(
+ "Setting up TezClassLoader: thread: {}, current thread classloader: {} system classloader: {}",
+ Thread.currentThread().getId(), Thread.currentThread().getContextClassLoader(),
+ ClassLoader.getSystemClassLoader());
Thread.currentThread().setContextClassLoader(INSTANCE);
}
+
+ public static void setupForConfiguration(Configuration configuration) {
+ configuration.setClassLoader(INSTANCE);
+ }
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 23811aa..3f02418 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -117,6 +117,7 @@ public class TezUtils {
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
Configuration conf = new Configuration(false);
readConfFromPB(confProto, conf);
+ TezClassLoader.setupForConfiguration(conf);
return conf;
}
}
@@ -130,6 +131,7 @@ public class TezUtils {
try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
readConfFromPB(confProto, configuration);
+ TezClassLoader.setupForConfiguration(configuration);
return configuration;
}
}
@@ -139,6 +141,7 @@ public class TezUtils {
try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
readConfFromPB(confProto, configuration);
+ TezClassLoader.setupForConfiguration(configuration);
}
}
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 6dd146a..86ceb12 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -77,6 +77,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import org.apache.tez.common.Preconditions;
+import org.apache.tez.common.TezClassLoader;
+
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
@@ -480,7 +482,7 @@ public class TezChild {
}
public static void main(String[] args) throws IOException, InterruptedException, TezException {
-
+ TezClassLoader.setupTezClassLoader();
final Configuration defaultConf = new Configuration();
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());