You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/20 15:06:46 UTC
ignite git commit: IGNITE-3941: Introduced common HadoopClassLoader.
This closes #1091.
Repository: ignite
Updated Branches:
refs/heads/ignite-1.6.8-hadoop 60c96ef3c -> 0f7ccb3d0
IGNITE-3941: Introduced common HadoopClassLoader. This closes #1091.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0f7ccb3d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0f7ccb3d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0f7ccb3d
Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 0f7ccb3d064cfa9be3cbd18bfa8bc74c32af52f0
Parents: 60c96ef
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Sep 20 18:06:39 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Sep 20 18:06:39 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 2 +-
.../processors/hadoop/HadoopClassLoader.java | 10 ---
.../processors/hadoop/HadoopHelper.java | 7 ++
.../processors/hadoop/HadoopNoopHelper.java | 17 +++++
.../hadoop/jobtracker/HadoopJobTracker.java | 10 +--
.../processors/hadoop/HadoopHelperImpl.java | 78 +++++++++++++-------
6 files changed, 76 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4f916be..c909de5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -822,7 +822,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
addHelper(IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration())));
- addHelper(HADOOP_HELPER.createIfInClassPath(null, false));
+ addHelper(HADOOP_HELPER.createIfInClassPath(ctx, false));
startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 3bf841b..90a2529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -36,7 +36,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
-import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -101,15 +100,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * Gets name for Job class loader. The name is specific for local node id.
- * @param locNodeId The local node id.
- * @return The class loader name.
- */
- public static String nameForJob(UUID locNodeId) {
- return "hadoop-job-node-" + locNodeId.toString();
- }
-
- /**
* Gets name for the task class loader. Task class loader
* @param info The task info.
* @param prefix Get only prefix (without task type and number)
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
index 40852c5..9297a2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
@@ -25,6 +25,13 @@ import org.jetbrains.annotations.Nullable;
*/
public interface HadoopHelper {
/**
+ * Get common Hadoop class loader.
+ *
+ * @return Common Hadoop class loader.
+ */
+ public HadoopClassLoader commonClassLoader();
+
+ /**
* Load special replacement and impersonate.
*
* @param in Input stream.
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
index d9ce857..32abc91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.hadoop;
+import org.apache.ignite.internal.GridKernalContext;
import org.jetbrains.annotations.Nullable;
import java.io.InputStream;
@@ -24,7 +25,23 @@ import java.io.InputStream;
/**
* Noop Hadoop Helper implementation.
*/
+@SuppressWarnings("unused")
public class HadoopNoopHelper implements HadoopHelper {
+ /**
+ * Constructor.
+ *
+ * @param ctx Kernal context.
+ */
+ @SuppressWarnings("UnusedParameters")
+ public HadoopNoopHelper(GridKernalContext ctx) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopClassLoader commonClassLoader() {
+ throw unsupported();
+ }
+
/** {@inheritDoc} */
@Override public boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr) {
throw unsupported();
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index a2c55a2..cb69823 100644
--- a/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -153,17 +153,9 @@ public class HadoopJobTracker extends HadoopComponent {
evtProcSvc = Executors.newFixedThreadPool(1);
- UUID nodeId = ctx.localNodeId();
-
assert jobCls == null;
- String[] libNames = null;
-
- if (ctx.configuration() != null)
- libNames = ctx.configuration().getNativeLibraryNames();
-
- HadoopClassLoader ldr = new HadoopClassLoader(null, HadoopClassLoader.nameForJob(nodeId), libNames,
- ctx.kernalContext().hadoopHelper());
+ HadoopClassLoader ldr = ctx.kernalContext().hadoopHelper().commonClassLoader();
try {
jobCls = (Class<HadoopV2Job>)ldr.loadClass(HadoopV2Job.class.getName());
http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
index 59a37e9..0ebe645 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.internal.processors.hadoop;
+import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.util.typedef.F;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
@@ -46,14 +47,53 @@ public class HadoopHelperImpl implements HadoopHelper {
/** Cache for resolved dependency info. */
private static final Map<String, Boolean> dependenciesCache = new ConcurrentHashMap8<>();
+ /** Kernal context. */
+ private final GridKernalContext ctx;
+
+ /** Common class loader. */
+ private volatile HadoopClassLoader ldr;
+
/**
- * Load special replacement and impersonate
+ * Default constructor.
+ */
+ public HadoopHelperImpl() {
+ this(null);
+ }
+
+ /**
+ * Constructor.
*
- * @param in Input stream.
- * @param originalName Original class name.
- * @param replaceName Replacer class name.
- * @return Result.
+ * @param ctx Kernal context.
*/
+ public HadoopHelperImpl(GridKernalContext ctx) {
+ this.ctx = ctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopClassLoader commonClassLoader() {
+ HadoopClassLoader res = ldr;
+
+ if (res == null) {
+ synchronized (this) {
+ res = ldr;
+
+ if (res == null) {
+ String[] libNames = null;
+
+ if (ctx != null && ctx.config().getHadoopConfiguration() != null)
+ libNames = ctx.config().getHadoopConfiguration().getNativeLibraryNames();
+
+ res = new HadoopClassLoader(null, "hadoop-common", libNames, this);
+
+ ldr = res;
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
@Override public byte[] loadReplace(InputStream in, final String originalName, final String replaceName) {
ClassReader rdr;
@@ -84,20 +124,12 @@ public class HadoopHelperImpl implements HadoopHelper {
return w.toByteArray();
}
- /**
- * @param cls Class name.
- * @return {@code true} If this is Hadoop class.
- */
+ /** {@inheritDoc} */
@Override public boolean isHadoop(String cls) {
return cls.startsWith("org.apache.hadoop.");
}
- /**
- * Need to parse only Ignite Hadoop and IGFS classes.
- *
- * @param cls Class name.
- * @return {@code true} if we need to check this class.
- */
+ /** {@inheritDoc} */
@Override public boolean isHadoopIgfs(String cls) {
String ignitePkgPrefix = "org.apache.ignite";
@@ -109,22 +141,12 @@ public class HadoopHelperImpl implements HadoopHelper {
cls.indexOf("hadoop.", len) != -1);
}
- /**
- * @param ldr Loader.
- * @param clsName Class.
- * @return Input stream.
- */
+ /** {@inheritDoc} */
@Override @Nullable public InputStream loadClassBytes(ClassLoader ldr, String clsName) {
return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
}
- /**
- * Check whether class has external dependencies on Hadoop.
- *
- * @param clsName Class name.
- * @param parentClsLdr Parent class loader.
- * @return {@code True} if class has external dependencies.
- */
+ /** {@inheritDoc} */
@Override public boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr) {
Boolean hasDeps = dependenciesCache.get(clsName);
@@ -152,7 +174,7 @@ public class HadoopHelperImpl implements HadoopHelper {
* @param ctx Context.
* @return {@code true} If the class has external dependencies.
*/
- boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr, CollectingContext ctx) {
+ private boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr, CollectingContext ctx) {
if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
return true;