You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/15 16:32:38 UTC
[1/2] ignite git commit: Finalization.
Repository: ignite
Updated Branches:
refs/heads/ignite-3909 5fa1e6068 -> a6ed71b2b
Finalization.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a68826dd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a68826dd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a68826dd
Branch: refs/heads/ignite-3909
Commit: a68826dd9fb0383e61f17bb72e26c90c3856bbca
Parents: 5fa1e60
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Sep 15 19:14:02 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Sep 15 19:14:02 2016 +0300
----------------------------------------------------------------------
.../src/main/java/com/mapr/DummyFileSystem.java | 99 ++++++++++++++++++++
.../processors/hadoop/HadoopClassLoader.java | 71 ++++++++++----
2 files changed, 150 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
new file mode 100644
index 0000000..63312c8
--- /dev/null
+++ b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
@@ -0,0 +1,99 @@
+package com.mapr;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URI;
+
+/**
+ * Created by vozerov on 9/15/2016.
+ */
+public class DummyFileSystem extends FileSystem {
+
+ public DummyFileSystem() throws IOException {
+ try {
+ Method m = UserGroupInformation.class.getDeclaredMethod("reset");
+
+ m.setAccessible(true);
+
+ m.invoke(null);
+
+ Field f = Groups.class.getDeclaredField("GROUPS");
+
+ f.setAccessible(true);
+
+ f.set(null, null);
+ }
+ catch (Exception ignore) {
+ throw new RuntimeException(ignore);
+ }
+
+ UserGroupInformation.getLoginUser();
+ }
+
+ @Override
+ public URI getUri() {
+ return null;
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ return null;
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
+ return null;
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ return false;
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
+ return new FileStatus[0];
+ }
+
+ @Override
+ public void setWorkingDirectory(Path new_dir) {
+
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return null;
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index c0b671e..f457ffa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -410,43 +410,74 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
String parentCls = clsName.substring(0, idx);
- // TODO: Looks suspicious.
- if (ctx.visited.contains(parentCls))
- return false;
-
return hasExternalDependencies(parentCls, ctx);
}
/**
+ * Check whether class name starts with particular prefix.
+ *
+ * @param cls Class name.
+ * @param skip Prefix to skip.
+ * @param prefix Prefix.
+ * @return {@code True} if starts with.
+ */
+ private static boolean startsWith(String cls, String skip, String prefix) {
+ assert cls.startsWith(skip);
+
+ return cls.startsWith(prefix, skip.length());
+ }
+
+ private static final String PKG_ORG = "org.";
+
+ private static final String PKG_ORG_APACHE = "org.apache.";
+
+ /**
* Whether we know in advance whether class has dependency or not.
*
* @param cls Class.
* @return Result.
*/
@Nullable private static Boolean hasDependencyPredefined(String cls) {
- // 1. Java systm classes never has dependencies.
- if (cls.startsWith("java.") || cls.startsWith("javax.") || cls.startsWith("sun.") || cls.startsWith("com.sun."))
- return false;
-
- // 2. Some other well-known packages.
- if (cls.startsWith("org.jsr166.") || cls.startsWith("org.w3c.") || cls.startsWith("org.xml.sax.") || cls.startsWith("org.slf4j.") || cls.startsWith("com.google.common."))
- return false;
+ // Large "org" group.
+ if (cls.startsWith(PKG_ORG)) {
+ // Large "apache" group.
+ if (cls.startsWith(PKG_ORG_APACHE)) {
+ // Hadoop classes always have dependencies.
+ if (startsWith(cls, PKG_ORG_APACHE, "hadoop."))
+ return true;
+
+ // Filter out Ignite classes which definitely do not have dependencies.
+ if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) {
+ if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs."))
+ return false;
+ }
- // 3. Special handling for "org.apache"
- if (cls.startsWith("org.apache.")) {
- if (cls.startsWith("org.apache.ignite")) {
- if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs."))
+ // Other well-known "org.apache" packages.
+ if (startsWith(cls, PKG_ORG_APACHE, "commons.") ||
+ startsWith(cls, PKG_ORG_APACHE, "log4j.") ||
+ startsWith(cls, PKG_ORG_APACHE, "xerces."))
return false;
}
- if (cls.startsWith("org.apache.hadoop"))
- return true;
-
- if (cls.startsWith("org.apache.xerces") || cls.startsWith("org.apache.log4j") || cls.startsWith("org.apache.commons.logging") || cls.startsWith("org.apache.commons.lang") || cls.startsWith("org.apache.commons.collections")
- || cls.startsWith("org.apache.commons.configuration"))
+ // Other well-known "org" packages.
+ if (startsWith(cls, PKG_ORG, "jsr166.") ||
+ startsWith(cls, PKG_ORG, "w3c.") ||
+ startsWith(cls, PKG_ORG, "slf4j.") ||
+ startsWith(cls, PKG_ORG, "xml.sax."))
return false;
}
+ // Filter out Java system packages.
+ if (cls.startsWith("java.") ||
+ cls.startsWith("javax.") ||
+ cls.startsWith("sun.") ||
+ cls.startsWith("com.sun."))
+ return false;
+
+ // Other well-known packages.
+ if (cls.startsWith("com.google.common"))
+ return false;
+
// No more guesses, will parse the class.
return null;
}
[2/2] ignite git commit: WIP.
Posted by vo...@apache.org.
WIP.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a6ed71b2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a6ed71b2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a6ed71b2
Branch: refs/heads/ignite-3909
Commit: a6ed71b2b594b763a57eb7b49bc08b42f374407c
Parents: a68826d
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Sep 15 19:32:25 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Sep 15 19:32:25 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 19 +++++++------------
1 file changed, 7 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a6ed71b2/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index f457ffa..4cc3aa4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -441,44 +441,39 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
// Large "org" group.
if (cls.startsWith(PKG_ORG)) {
// Large "apache" group.
- if (cls.startsWith(PKG_ORG_APACHE)) {
+ if (startsWith(cls, PKG_ORG, "apache.")) {
// Hadoop classes always have dependencies.
if (startsWith(cls, PKG_ORG_APACHE, "hadoop."))
return true;
-
// Filter out Ignite classes which definitely do not have dependencies.
- if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) {
+ else if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) {
if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs."))
return false;
}
-
// Other well-known "org.apache" packages.
- if (startsWith(cls, PKG_ORG_APACHE, "commons.") ||
+ else if (startsWith(cls, PKG_ORG_APACHE, "commons.") ||
startsWith(cls, PKG_ORG_APACHE, "log4j.") ||
startsWith(cls, PKG_ORG_APACHE, "xerces."))
return false;
}
-
// Other well-known "org" packages.
- if (startsWith(cls, PKG_ORG, "jsr166.") ||
+ else if (startsWith(cls, PKG_ORG, "jsr166.") ||
startsWith(cls, PKG_ORG, "w3c.") ||
startsWith(cls, PKG_ORG, "slf4j.") ||
startsWith(cls, PKG_ORG, "xml.sax."))
return false;
}
-
// Filter out Java system packages.
- if (cls.startsWith("java.") ||
+ else if (cls.startsWith("java.") ||
cls.startsWith("javax.") ||
cls.startsWith("sun.") ||
cls.startsWith("com.sun."))
return false;
-
// Other well-known packages.
- if (cls.startsWith("com.google.common"))
+ else if (cls.startsWith("com.google.common"))
return false;
- // No more guesses, will parse the class.
+ // Will parse class.
return null;
}