You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ha...@apache.org on 2020/08/13 05:33:40 UTC
[tez] branch branch-0.9 updated: TEZ-4223 - Adding new jars or
resources after the first DAG runs does not work.
This is an automated email from the ASF dual-hosted git repository.
harishjp pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 774bce1 TEZ-4223 - Adding new jars or resources after the first DAG runs does not work.
774bce1 is described below
commit 774bce1c34501f94542dde866efd0ff06bcaaabc
Author: Harish JP <ha...@apache.org>
AuthorDate: Thu Aug 13 08:57:35 2020 +0530
TEZ-4223 - Adding new jars or resources after the first DAG runs does not work.
Signed-off-by: Rajesh Balamohan <rb...@apache.org>
(cherry picked from commit 6fc75ad6e9b1601b8b14dd85fa9b0aea53585fba)
---
.../org/apache/tez/common/ReflectionUtils.java | 43 +--------------------
.../java/org/apache/tez/common/TezClassLoader.java | 45 ++++++++--------------
.../org/apache/tez/common/TestReflectionUtils.java | 4 +-
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +
4 files changed, 21 insertions(+), 73 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
index 9f7c5d3..73becda 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java
@@ -19,17 +19,14 @@
package org.apache.tez.common;
import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
-import java.net.URLClassLoader;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.tez.dag.api.TezReflectionException;
-import org.apache.tez.dag.api.TezUncheckedException;
@Private
public class ReflectionUtils {
@@ -110,46 +107,10 @@ public class ReflectionUtils {
}
@Private
- public static synchronized void addResourcesToClasspath(List<URL> urls) {
- ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Thread
- .currentThread().getContextClassLoader());
- Thread.currentThread().setContextClassLoader(classLoader);
- }
-
- // Parameters for addResourcesToSystemClassLoader
- private static final Class<?>[] parameters = new Class[]{URL.class};
- private static Method sysClassLoaderMethod = null;
-
- @Private
public static synchronized void addResourcesToSystemClassLoader(List<URL> urls) {
- ClassLoader sysLoader = getSystemClassLoader();
- if (sysClassLoaderMethod == null) {
- Class<?> sysClass = TezClassLoader.class;
- Method method;
- try {
- method = sysClass.getDeclaredMethod("addURL", parameters);
- } catch (SecurityException e) {
- throw new TezUncheckedException("Failed to get handle on method addURL", e);
- } catch (NoSuchMethodException e) {
- throw new TezUncheckedException("Failed to get handle on method addURL", e);
- }
- method.setAccessible(true);
- sysClassLoaderMethod = method;
- }
+ TezClassLoader classLoader = TezClassLoader.getInstance();
for (URL url : urls) {
- try {
- sysClassLoaderMethod.invoke(sysLoader, new Object[] { url });
- } catch (IllegalArgumentException e) {
- throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
- } catch (IllegalAccessException e) {
- throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
- } catch (InvocationTargetException e) {
- throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
- }
+ classLoader.addURL(url);
}
}
-
- private static ClassLoader getSystemClassLoader() {
- return TezClassLoader.getInstance();
- }
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
index 2679efa..1842a19 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java
@@ -13,30 +13,24 @@
*/
package org.apache.tez.common;
-import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Arrays;
+/**
+ * ClassLoader to allow addition of new paths to classpath in the runtime.
+ *
+ * It uses URLClassLoader with this class' classloader as parent classloader.
+ * And hence first delegates the resource loading to parent and then to the URLs
+ * added. The process must be setup to use by invoking setupTezClassLoader() which sets
+ * the global TezClassLoader as current thread context class loader. All threads
+ * created will inherit the classloader and hence will resolve the class/resource
+ * from TezClassLoader.
+ */
public class TezClassLoader extends URLClassLoader {
- private static TezClassLoader INSTANCE;
-
- static {
- INSTANCE = AccessController.doPrivileged(new PrivilegedAction<TezClassLoader>() {
- ClassLoader sysLoader = TezClassLoader.class.getClassLoader();
-
- public TezClassLoader run() {
- return new TezClassLoader(
- sysLoader instanceof URLClassLoader ? ((URLClassLoader) sysLoader).getURLs() : extractClassPathEntries(),
- sysLoader);
- }
- });
- }
+ private static TezClassLoader INSTANCE = new TezClassLoader();
- public TezClassLoader(URL[] urls, ClassLoader classLoader) {
- super(urls, classLoader);
+ private TezClassLoader() {
+ super(new URL[] {}, TezClassLoader.class.getClassLoader());
}
public void addURL(URL url) {
@@ -47,16 +41,7 @@ public class TezClassLoader extends URLClassLoader {
return INSTANCE;
}
- private static URL[] extractClassPathEntries() {
- String pathSeparator = System.getProperty("path.separator");
- String[] classPathEntries = System.getProperty("java.class.path").split(pathSeparator);
- URL[] cp = Arrays.asList(classPathEntries).stream().map(s -> {
- try {
- return new URL("file://" + s);
- } catch (MalformedURLException e) {
- throw new RuntimeException(e);
- }
- }).toArray(URL[]::new);
- return cp;
+ public static void setupTezClassLoader() {
+ Thread.currentThread().setContextClassLoader(INSTANCE);
}
}
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
index 2fbd35c..ed3814d 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java
@@ -58,7 +58,7 @@ public class TestReflectionUtils {
@Test(timeout = 5000)
public void testAddResourceToClasspath() throws IOException, TezException {
-
+ TezClassLoader.setupTezClassLoader();
String rsrcName = "dummyfile.xml";
FileSystem localFs = FileSystem.getLocal(new Configuration());
Path p = new Path(rsrcName);
@@ -78,7 +78,7 @@ public class TestReflectionUtils {
urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1);
URL url = new URL(urlForm);
- ReflectionUtils.addResourcesToClasspath(Collections.singletonList(url));
+ ReflectionUtils.addResourcesToSystemClassLoader(Collections.singletonList(url));
loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName);
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index af62d9d..d70573a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2387,6 +2387,8 @@ public class DAGAppMaster extends AbstractService {
public static void main(String[] args) {
try {
+ // Install the tez class loader, which can be used add new resources
+ TezClassLoader.setupTezClassLoader();
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
final String pid = System.getenv().get("JVM_PID");
String containerIdStr =