You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/20 13:55:12 UTC
[3/9] ignite git commit: 3929: wip, current state.
3929: wip, current state.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83ff85f3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83ff85f3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83ff85f3
Branch: refs/heads/ignite-3929-1
Commit: 83ff85f3babd3bedb70fda86703146d2ada393a0
Parents: df3ad76
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Sep 20 16:03:35 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Sep 20 16:03:35 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopSnappyTest.java | 2 +-
.../processors/hadoop/HadoopClassLoader.java | 51 ++++++++++++++------
.../internal/processors/hadoop/LoadHelper.java | 31 +++++++-----
3 files changed, 56 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
index 0b86c50..ca47334 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
@@ -58,7 +58,7 @@ public class HadoopSnappyTest extends GridCommonAbstractTest {
U.invoke(cls, null, "checkSnappy");
}
- //checkSnappy();
+ checkSnappy();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index c6d2296..276b22d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.hadoop;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -114,6 +115,8 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return "hadoop-task-" + info.jobId() + "-" + info.type() + "-" + info.taskNumber();
}
+ private static final String[] DEFAULT_OPTIONAL_LIBS_TO_BE_LOADED = { "hadoop" };
+
/**
* Constructor.
*
@@ -202,20 +205,30 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
* be loaded only once.
*/
private static void setNativeLibrariesToBeInjectedIfNeeded(HadoopClassLoader instance, String[] libs) {
+ System.out.println("### Loading libs: " + libs);
+
if (libs == null)
return;
-
- boolean created = false;
+//
+// boolean created = false;
// 1. If needed, init the native lib data collection:
if (nativeLibrariesToBeInjected == null) {
synchronized (HadoopClassLoader.class) {
if (nativeLibrariesToBeInjected == null) {
+ LinkedHashSet<Object> natives0 = getNativeLibraries(APP_CLS_LDR);
+
instance.runLoadingCode(libs);
- nativeLibrariesToBeInjected = instance.collectNativeLibraries();
+ LinkedHashSet<Object> natives1 = getNativeLibraries(APP_CLS_LDR);
- created = true;
+ natives1.removeAll(natives0);
+
+ nativeLibrariesToBeInjected = Collections.unmodifiableCollection(natives1);
+
+ System.out.println("### Collected loaded libraries: " + nativeLibrariesToBeInjected);
+//
+// created = true;
}
}
}
@@ -223,9 +236,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
assert nativeLibrariesToBeInjected != null;
// 2. Inject libraries:
- if (!created)
+ //if (!created)
// This is an instance that did not load the libs:
- instance.injectNatives();
+ instance.injectNatives();
+ }
+
+ /**
+ *
+ * @return
+ */
+ private static LinkedHashSet<Object> getNativeLibraries(ClassLoader cl) {
+ Vector<Object> curVector = U.field(cl, "nativeLibraries");
+
+ return new LinkedHashSet<>(curVector);
}
/**
@@ -251,7 +274,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
private Collection<Object> collectNativeLibraries() {
List<Object> target = new ArrayList<>();
- ClassLoader ldr = this;
+ ClassLoader ldr = APP_CLS_LDR;
while (ldr != null) {
collectNativeLibrariesFromLoader(ldr, target);
@@ -267,17 +290,17 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
*/
private void runLoadingCode(String[] libs) {
try {
- // TODO: "XXX" is a special class loaded by Hadoop class loader (simulating Hadoop class).
- // NB: this sample class must *not* cause loading of any natives.
- Class<?> sampleCls = this.loadClass(XXX.class.getName(), true);
-
- assert sampleCls != null;
- assert sampleCls.getClassLoader() == this;
+// // TODO: "XXX" is a special class loaded by Hadoop class loader (simulating Hadoop class).
+// // NB: this sample class must *not* cause loading of any natives.
+// Class<?> sampleCls = this.loadClass(XXX.class.getName(), true);
+//
+// assert sampleCls != null;
+// assert sampleCls.getClassLoader() == this;
Collection<String> loadedLibs = new ArrayList<>();
for (String lib: libs) {
- boolean ok = LoadHelper.tryLoad(sampleCls, lib);
+ boolean ok = LoadHelper.tryLoad(null, lib);
if (ok)
loadedLibs.add(lib);
http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
index a795085..223855b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
@@ -7,18 +7,18 @@ import java.lang.reflect.Method;
*/
public class LoadHelper {
- private static Method method;
-
- static {
- try {
- method = ClassLoader.class.getDeclaredMethod("loadLibrary",
- new Class[] {Class.class, String.class, boolean.class});
-
- method.setAccessible(true);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+// private static Method method;
+//
+// static {
+// try {
+// method = ClassLoader.class.getDeclaredMethod("loadLibrary",
+// new Class[] {Class.class, String.class, boolean.class});
+//
+// method.setAccessible(true);
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// }
/**
* Utility method that loads given class by name with the given "caller" class.
@@ -27,7 +27,7 @@ public class LoadHelper {
*/
public static boolean tryLoad(Class caller, String libName) {
try {
- method.invoke(null, new Object[] {caller/*caller class*/, libName /*lib*/ , false/*isAbsolute*/ });
+ System.loadLibrary(libName);
return true;
}
@@ -39,4 +39,9 @@ public class LoadHelper {
}
}
+
+ public static void main(String[] args) {
+ System.load("/home/ivan/hadoop-2.6.0/lib/native/libhadoop.so");
+ }
+
}