You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/15 15:46:42 UTC
[6/7] ignite git commit: WIP on fixes.
WIP on fixes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8e5dcd2f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8e5dcd2f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8e5dcd2f
Branch: refs/heads/ignite-3909
Commit: 8e5dcd2f0f4d9da3585e5aa3c2ef4506db23d362
Parents: 2474e2b
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Sep 15 18:27:11 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Sep 15 18:27:11 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 161 ++++++++++---------
.../internal/processors/hadoop/HadoopUtils.java | 3 +
.../processors/hadoop/v2/HadoopV2Job.java | 2 +-
.../hadoop/HadoopClassLoaderTest.java | 4 +-
4 files changed, 94 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 2e0e271..a921f03 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -61,13 +61,11 @@ import org.objectweb.asm.commons.RemappingClassAdapter;
* unavailable for parent.
*/
public class HadoopClassLoader extends URLClassLoader implements ClassCache {
- static {
- // We are very parallel capable.
- registerAsParallelCapable();
- }
+ /** Name of the Hadoop shutdown hook manager class. */
+ public static final String SHUTDOWN_HOOK_MGR_CLS_NAME = "org.apache.hadoop.util.ShutdownHookManager";
/** Name of the Hadoop daemon class. */
- public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon";
+ public static final String HADOOP_DAEMON_CLS_NAME = "org.apache.hadoop.util.Daemon";
/** Name of libhadoop library. */
private static final String LIBHADOOP = "hadoop.";
@@ -97,6 +95,11 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
/** Native library names. */
private final String[] libNames;
+ static {
+ // We are very parallel capable.
+ registerAsParallelCapable();
+ }
+
/**
* Gets name for Job class loader. The name is specific for local node id.
* @param locNodeId The local node id.
@@ -192,57 +195,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
}
- /**
- * Need to parse only Ignite Hadoop and IGFS classes.
- *
- * @param cls Class name.
- * @return {@code true} if we need to check this class.
- */
- private static boolean isHadoopIgfs(String cls) {
- String ignitePkgPrefix = "org.apache.ignite";
-
- int len = ignitePkgPrefix.length();
-
- return cls.startsWith(ignitePkgPrefix) && (
- cls.indexOf("igfs.", len) != -1 ||
- cls.indexOf(".fs.", len) != -1 ||
- cls.indexOf("hadoop.", len) != -1);
- }
-
- /**
- * @param cls Class name.
- * @return {@code true} If this is Hadoop class.
- */
- private static boolean isHadoop(String cls) {
- return cls.startsWith("org.apache.hadoop.");
- }
-
/** {@inheritDoc} */
@Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
try {
- if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
- if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks.
- return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
- else if (name.equals(HADOOP_DAEMON_CLASS_NAME))
- // We replace this in order to be able to forcibly stop some daemon threads
- // that otherwise never stop (e.g. PeerCache runnables):
- return loadFromBytes(name, HadoopDaemon.class.getName());
-
+ if (name.equals(SHUTDOWN_HOOK_MGR_CLS_NAME))
+ // Dirty hack to get rid of Hadoop shutdown hooks.
+ return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
+ else if (name.equals(HADOOP_DAEMON_CLS_NAME))
+ // We replace this in order to be able to forcibly stop some daemon threads
+ // that otherwise never stop (e.g. PeerCache runnables):
+ return loadFromBytes(name, HadoopDaemon.class.getName());
+
+ if (hasExternalDependencies(name))
return loadClassExplicitly(name, resolve);
- }
-
- if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
- Boolean hasDeps = cache.get(name);
-
- if (hasDeps == null) {
- hasDeps = hasExternalDependencies(name);
-
- cache.put(name, hasDeps);
- }
-
- if (hasDeps)
- return loadClassExplicitly(name, resolve);
- }
return super.loadClass(name, resolve);
}
@@ -362,14 +327,39 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
* @return {@code True} if class has external dependencies.
*/
boolean hasExternalDependencies(String clsName) {
- CollectingContext ctx = new CollectingContext();
+ Boolean res = cache.get(clsName);
+
+ if (res == null) {
+ CollectingContext ctx = new CollectingContext();
+
+ ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
+ ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
+ ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
+ ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor);
+
+ return hasExternalDependencies(clsName, ctx);
+ }
+ else
+ return res;
+ }
+
+ /**
+ * Check whether class has external dependencies on Hadoop.
+ *
+ * @param clsName Class name.
+ * @param ctx Context.
+ * @return {@code True} if class has external dependencies.
+ */
+ private boolean hasExternalDependencies(String clsName, CollectingContext ctx) {
+ Boolean res = cache.get(clsName);
- ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
- ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
- ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
- ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor);
+ if (res == null) {
+ res = hasExternalDependencies0(clsName, ctx);
- return hasExternalDependencies(clsName, ctx);
+ cache.put(clsName, res);
+ }
+
+ return res;
}
/**
@@ -379,9 +369,13 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
* @param ctx Context.
* @return {@code true} If the class has external dependencies.
*/
- boolean hasExternalDependencies(String clsName, CollectingContext ctx) {
- if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
- return true;
+ private boolean hasExternalDependencies0(String clsName, CollectingContext ctx) {
+ Boolean res = hasDependencyPredefined(clsName);
+
+ if (res != null)
+ return res;
+
+ System.out.println(">>> EXT: " + clsName);
// Try to get from parent to check if the type accessible.
InputStream in = loadClassBytes(getParent(), clsName);
@@ -389,16 +383,16 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
if (in == null) // The class is external itself, it must be loaded from this class loader.
return true;
- if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies.
- return false;
-
final ClassReader rdr;
try {
rdr = new ClassReader(in);
}
- catch (IOException e) {
- throw new RuntimeException("Failed to read class: " + clsName, e);
+ catch (Exception e) {
+ System.out.println(">>> ERR: " + clsName);
+
+ return false;
+ //throw new RuntimeException("Failed to read class: " + clsName, e);
}
ctx.visited.add(clsName);
@@ -416,15 +410,39 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
String parentCls = clsName.substring(0, idx);
+ // TODO: Looks suspicious.
if (ctx.visited.contains(parentCls))
return false;
- Boolean res = cache.get(parentCls);
+ return hasExternalDependencies(parentCls, ctx);
+ }
+
+ /**
+ * Whether we know in advance whether class has dependency or not.
+ *
+ * @param cls Class.
+ * @return Result.
+ */
+ @Nullable private static Boolean hasDependencyPredefined(String cls) {
+ // 1. Java systm classes never has dependencies.
+ if (cls.startsWith("java.") || cls.startsWith("javax.") || cls.startsWith("sun.") || cls.startsWith("com.sun."))
+ return false;
+
+ // 2. Some other well-known packages.
+ if (cls.startsWith("org.jsr166.") || cls.startsWith("org.w3c.") || cls.startsWith("org.xml.sax.") || cls.startsWith("org.slf4j.") || cls.startsWith("com.google.common."))
+ return false;
- if (res == null)
- res = hasExternalDependencies(parentCls, ctx);
+ // 3. Special handling for "org.apache"
+ if (cls.startsWith("org.apache.")) {
+ if (cls.startsWith("org.apache.ignite"))
+ return cls.contains(".hadoop.") || cls.contains(".igfs.") || cls.contains(".fs.");
- return res;
+ if (cls.startsWith("org.apache.hadoop"))
+ return true;
+ }
+
+ // No more guesses, will parse the class.
+ return null;
}
/**
@@ -572,9 +590,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
assert depCls.charAt(0) != 'L' : depCls;
assert validateClassName(depCls) : depCls;
- if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes.
- return;
-
if (visited.contains(depCls))
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index 65d9810..6d3866b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -277,6 +277,9 @@ public class HadoopUtils {
for (Map.Entry<String, String> entry : jobConf)
props.put(entry.getKey(), entry.getValue());
+ props.put("fs.defaultFS", "dummy:///test/");
+ props.put("fs.dummy.impl", "com.mapr.DummyFileSystem");
+
return new HadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces, props);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index a0f30eb..1264e48 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -382,7 +382,7 @@ public class HadoopV2Job implements HadoopJob {
* @throws Exception On error.
*/
private void stopHadoopFsDaemons(ClassLoader ldr) throws Exception {
- Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLASS_NAME);
+ Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLS_NAME);
Method m = daemonCls.getMethod("dequeueAndStopAll");
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e5dcd2f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
index 02d98d0..2fd7777 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
@@ -94,7 +94,7 @@ public class HadoopClassLoaderTest extends TestCase {
CircularWithoutHadoop.class,
};
- for (Class c: positiveClasses)
+ for (Class c : positiveClasses)
assertTrue(c.getName(), ldr.hasExternalDependencies(c.getName()));
// Negative cases:
@@ -104,7 +104,7 @@ public class HadoopClassLoaderTest extends TestCase {
Without.class,
};
- for (Class c: negativeClasses)
+ for (Class c : negativeClasses)
assertFalse(c.getName(), ldr.hasExternalDependencies(c.getName()));
}
}
\ No newline at end of file