You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ha...@apache.org on 2020/08/13 05:48:05 UTC
[tez] branch branch-0.9 updated: Revert "TEZ-4223 - Adding new jars
or resources after the first DAG runs does not work."
This is an automated email from the ASF dual-hosted git repository.
harishjp pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 4b1b207 Revert "TEZ-4223 - Adding new jars or resources after the first DAG runs does not work."
4b1b207 is described below
commit 4b1b207269f90da907b8f38d8c6960c3ac096503
Author: Harish JP <ha...@apache.org>
AuthorDate: Thu Aug 13 11:17:37 2020 +0530
Revert "TEZ-4223 - Adding new jars or resources after the first DAG runs does not work."
This reverts commit 774bce1c34501f94542dde866efd0ff06bcaaabc.
---
.../org/apache/tez/common/ReflectionUtils.java | 43 ++++++++++++++++++++-
.../java/org/apache/tez/common/TezClassLoader.java | 45 ++++++++++++++--------
.../org/apache/tez/common/TestReflectionUtils.java | 4 +-
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 -
4 files changed, 73 insertions(+), 21 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
index 73becda..9f7c5d3 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
@@ -19,14 +19,17 @@
package org.apache.tez.common;
import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
+import java.net.URLClassLoader;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.tez.dag.api.TezReflectionException;
+import org.apache.tez.dag.api.TezUncheckedException;
@Private
public class ReflectionUtils {
@@ -107,10 +110,46 @@ public class ReflectionUtils {
}
@Private
+ public static synchronized void addResourcesToClasspath(List<URL> urls) {
+ ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Thread
+ .currentThread().getContextClassLoader());
+ Thread.currentThread().setContextClassLoader(classLoader);
+ }
+
+ // Parameters for addResourcesToSystemClassLoader
+ private static final Class<?>[] parameters = new Class[]{URL.class};
+ private static Method sysClassLoaderMethod = null;
+
+ @Private
public static synchronized void addResourcesToSystemClassLoader(List<URL> urls) {
- TezClassLoader classLoader = TezClassLoader.getInstance();
+ ClassLoader sysLoader = getSystemClassLoader();
+ if (sysClassLoaderMethod == null) {
+ Class<?> sysClass = TezClassLoader.class;
+ Method method;
+ try {
+ method = sysClass.getDeclaredMethod("addURL", parameters);
+ } catch (SecurityException e) {
+ throw new TezUncheckedException("Failed to get handle on method addURL", e);
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException("Failed to get handle on method addURL", e);
+ }
+ method.setAccessible(true);
+ sysClassLoaderMethod = method;
+ }
for (URL url : urls) {
- classLoader.addURL(url);
+ try {
+ sysClassLoaderMethod.invoke(sysLoader, new Object[] { url });
+ } catch (IllegalArgumentException e) {
+ throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
+ }
}
}
+
+ private static ClassLoader getSystemClassLoader() {
+ return TezClassLoader.getInstance();
+ }
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
index 1842a19..2679efa 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
@@ -13,24 +13,30 @@
*/
package org.apache.tez.common;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
-/**
- * ClassLoader to allow addition of new paths to classpath in the runtime.
- *
- * It uses URLClassLoader with this class' classloader as parent classloader.
- * And hence first delegates the resource loading to parent and then to the URLs
- * added. The process must be setup to use by invoking setupTezClassLoader() which sets
- * the global TezClassLoader as current thread context class loader. All threads
- * created will inherit the classloader and hence will resolve the class/resource
- * from TezClassLoader.
- */
public class TezClassLoader extends URLClassLoader {
- private static TezClassLoader INSTANCE = new TezClassLoader();
+ private static TezClassLoader INSTANCE;
+
+ static {
+ INSTANCE = AccessController.doPrivileged(new PrivilegedAction<TezClassLoader>() {
+ ClassLoader sysLoader = TezClassLoader.class.getClassLoader();
+
+ public TezClassLoader run() {
+ return new TezClassLoader(
+ sysLoader instanceof URLClassLoader ? ((URLClassLoader) sysLoader).getURLs() : extractClassPathEntries(),
+ sysLoader);
+ }
+ });
+ }
- private TezClassLoader() {
- super(new URL[] {}, TezClassLoader.class.getClassLoader());
+ public TezClassLoader(URL[] urls, ClassLoader classLoader) {
+ super(urls, classLoader);
}
public void addURL(URL url) {
@@ -41,7 +47,16 @@ public class TezClassLoader extends URLClassLoader {
return INSTANCE;
}
- public static void setupTezClassLoader() {
- Thread.currentThread().setContextClassLoader(INSTANCE);
+ private static URL[] extractClassPathEntries() {
+ String pathSeparator = System.getProperty("path.separator");
+ String[] classPathEntries = System.getProperty("java.class.path").split(pathSeparator);
+ URL[] cp = Arrays.asList(classPathEntries).stream().map(s -> {
+ try {
+ return new URL("file://" + s);
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ }).toArray(URL[]::new);
+ return cp;
}
}
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
index ed3814d..2fbd35c 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
@@ -58,7 +58,7 @@ public class TestReflectionUtils {
@Test(timeout = 5000)
public void testAddResourceToClasspath() throws IOException, TezException {
- TezClassLoader.setupTezClassLoader();
+
String rsrcName = "dummyfile.xml";
FileSystem localFs = FileSystem.getLocal(new Configuration());
Path p = new Path(rsrcName);
@@ -78,7 +78,7 @@ public class TestReflectionUtils {
urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1);
URL url = new URL(urlForm);
- ReflectionUtils.addResourcesToSystemClassLoader(Collections.singletonList(url));
+ ReflectionUtils.addResourcesToClasspath(Collections.singletonList(url));
loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName);
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index d70573a..af62d9d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2387,8 +2387,6 @@ public class DAGAppMaster extends AbstractService {
public static void main(String[] args) {
try {
- // Install the tez class loader, which can be used add new resources
- TezClassLoader.setupTezClassLoader();
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
final String pid = System.getenv().get("JVM_PID");
String containerIdStr =