You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/16 09:55:12 UTC
[1/2] ignite git commit: IGNITE-3912: Preliminary changes.
Repository: ignite
Updated Branches:
refs/heads/ignite-1.6.8-hadoop [created] 2fe0272ba
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
deleted file mode 100644
index f5c2814..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ /dev/null
@@ -1,461 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Hadoop classpath utilities.
- */
-public class HadoopClasspathUtils {
- /** Prefix directory. */
- public static final String PREFIX = "HADOOP_PREFIX";
-
- /** Hadoop home directory. */
- public static final String HOME = "HADOOP_HOME";
-
- /** Hadoop common directory. */
- public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
-
- /** Hadoop HDFS directory. */
- public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
-
- /** Hadoop mapred directory. */
- public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
-
- /** Arbitrary additional dependencies. Compliant with standard Java classpath resolution. */
- public static final String HADOOP_USER_LIBS = "HADOOP_USER_LIBS";
-
- /** Empty string. */
- private static final String EMPTY_STR = "";
-
- /**
- * Gets Hadoop class path as list of classpath elements for process.
- *
- * @return List of the class path elements.
- * @throws IOException If failed.
- */
- public static List<String> classpathForProcess() throws IOException {
- List<String> res = new ArrayList<>();
-
- for (final SearchDirectory dir : classpathDirectories()) {
- File[] files = dir.files();
-
- if (dir.useWildcard()) {
- if (files.length > 0)
- res.add(dir.absolutePath() + File.separator + '*');
- }
- else {
- for (File file : files)
- res.add(file.getAbsolutePath());
- }
- }
-
- return res;
- }
-
- /**
- * Gets Hadoop class path as a list of URLs (for in-process class loader usage).
- *
- * @return List of class path URLs.
- * @throws IOException If failed.
- */
- public static List<URL> classpathForClassLoader() throws IOException {
- List<URL> res = new ArrayList<>();
-
- for (SearchDirectory dir : classpathDirectories()) {
- for (File file : dir.files()) {
- try {
- res.add(file.toURI().toURL());
- }
- catch (MalformedURLException e) {
- throw new IOException("Failed to convert file path to URL: " + file.getPath());
- }
- }
- }
-
- return res;
- }
-
- /**
- * Gets Hadoop locations.
- *
- * @return The locations as determined from the environment.
- */
- public static HadoopLocations locations() throws IOException {
- // Query environment.
- String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR));
-
- String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR);
- String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR);
- String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
-
- // If any composite location is defined, use only them.
- if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) {
- HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
-
- if (res.valid())
- return res;
- else
- throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " +
- "either undefined or point to nonexistent directories [" +
- "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " +
- "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
- "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
- }
- else if (!isEmpty(hadoopHome)) {
- // All further checks will be based on HADOOP_HOME, so check for it's existence.
- if (!exists(hadoopHome))
- throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
- "variable points to nonexistent directory: " + hadoopHome);
-
- // Probe Apache Hadoop.
- HadoopLocations res = new HadoopLocations(
- hadoopHome,
- hadoopHome + "/share/hadoop/common",
- hadoopHome + "/share/hadoop/hdfs",
- hadoopHome + "/share/hadoop/mapreduce"
- );
-
- if (res.valid())
- return res;
-
- // Probe CDH.
- res = new HadoopLocations(
- hadoopHome,
- hadoopHome,
- hadoopHome + "/../hadoop-hdfs",
- hadoopHome + "/../hadoop-mapreduce"
- );
-
- if (res.valid())
- return res;
-
- // Probe HDP.
- res = new HadoopLocations(
- hadoopHome,
- hadoopHome,
- hadoopHome + "/../hadoop-hdfs-client",
- hadoopHome + "/../hadoop-mapreduce-client"
- );
-
- if (res.valid())
- return res;
-
- // Failed.
- throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
- "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome);
- }
- else {
- // Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
- throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " +
- "variable and point it to your Hadoop distribution).");
- }
- }
-
- /**
- * Gets base directories to discover classpath elements in.
- *
- * @return Collection of directory and mask pairs.
- * @throws IOException if a mandatory classpath location is not found.
- */
- private static Collection<SearchDirectory> classpathDirectories() throws IOException {
- HadoopLocations loc = locations();
-
- Collection<SearchDirectory> res = new ArrayList<>();
-
- res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
- res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
- res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
-
- res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-common-")));
- res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-auth-")));
-
- res.add(new SearchDirectory(new File(loc.hdfs()), new PrefixDirectoryFilter("hadoop-hdfs-")));
-
- res.add(new SearchDirectory(new File(loc.mapred()),
- new PrefixDirectoryFilter("hadoop-mapreduce-client-common")));
- res.add(new SearchDirectory(new File(loc.mapred()),
- new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
-
- res.addAll(parseUserLibs());
-
- return res;
- }
-
- /**
- * Parse user libs.
- *
- * @return Parsed libs search patterns.
- * @throws IOException If failed.
- */
- static Collection<SearchDirectory> parseUserLibs() throws IOException {
- return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null));
- }
-
- /**
- * Parse user libs.
- *
- * @param str String.
- * @return Result.
- * @throws IOException If failed.
- */
- static Collection<SearchDirectory> parseUserLibs(String str) throws IOException {
- Collection<SearchDirectory> res = new LinkedList<>();
-
- if (!isEmpty(str)) {
- String[] tokens = normalize(str).split(File.pathSeparator);
-
- for (String token : tokens) {
- // Skip empty tokens.
- if (isEmpty(token))
- continue;
-
- File file = new File(token);
- File dir = file.getParentFile();
-
- if (token.endsWith("*")) {
- assert dir != null;
-
- res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, false));
- }
- else {
- // Met "/" or "C:\" pattern, nothing to do with it.
- if (dir == null)
- continue;
-
- res.add(new SearchDirectory(dir, new ExactDirectoryFilter(file.getName()), false));
- }
- }
- }
-
- return res;
- }
-
- /**
- * Get system property or environment variable with the given name.
- *
- * @param name Variable name.
- * @param dflt Default value.
- * @return Value.
- */
- private static String systemOrEnv(String name, String dflt) {
- String res = System.getProperty(name);
-
- if (res == null)
- res = System.getenv(name);
-
- return res != null ? res : dflt;
- }
-
- /**
- * Answers if the given path denotes existing directory.
- *
- * @param path The directory path.
- * @return {@code True} if the given path denotes an existing directory.
- */
- public static boolean exists(String path) {
- if (path == null)
- return false;
-
- Path p = Paths.get(path);
-
- return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
- }
-
- /**
- * Check if string is empty.
- *
- * @param val Value.
- * @return {@code True} if empty.
- */
- private static boolean isEmpty(String val) {
- return val == null || val.isEmpty();
- }
-
- /**
- * NOramlize the string.
- *
- * @param str String.
- * @return Normalized string.
- */
- private static String normalize(String str) {
- assert str != null;
-
- return str.trim().toLowerCase();
- }
-
- /**
- * Simple pair-like structure to hold directory name and a mask assigned to it.
- */
- static class SearchDirectory {
- /** File. */
- private final File dir;
-
- /** Filter. */
- private final DirectoryFilter filter;
-
- /** Whether directory must exist. */
- private final boolean strict;
-
- /**
- * Constructor for directory search with strict rule.
- *
- * @param dir Directory.
- * @param filter Filter.
- * @throws IOException If failed.
- */
- private SearchDirectory(File dir, DirectoryFilter filter) throws IOException {
- this(dir, filter, true);
- }
-
- /**
- * Constructor.
- *
- * @param dir Directory.
- * @param filter Filter.
- * @param strict Whether directory must exist.
- * @throws IOException If failed.
- */
- private SearchDirectory(File dir, DirectoryFilter filter, boolean strict) throws IOException {
- this.dir = dir;
- this.filter = filter;
- this.strict = strict;
-
- if (strict && !exists(dir.getAbsolutePath()))
- throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
- }
-
- /**
- * @return Absolute path.
- */
- String absolutePath() {
- return dir.getAbsolutePath();
- }
-
- /**
- * @return Child files.
- */
- File[] files() throws IOException {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return filter.test(name);
- }
- });
-
- if (files == null) {
- if (strict)
- throw new IOException("Failed to get directory files [dir=" + dir + ']');
- else
- return new File[0];
- }
- else
- return files;
- }
-
- /**
- * @return {@code True} if wildcard can be used.
- */
- boolean useWildcard() {
- return filter instanceof AcceptAllDirectoryFilter;
- }
- }
-
- /**
- * Directory filter interface.
- */
- static interface DirectoryFilter {
- /**
- * Test if file with this name should be included.
- *
- * @param name File name.
- * @return {@code True} if passed.
- */
- public boolean test(String name);
- }
-
- /**
- * Filter to accept all files.
- */
- static class AcceptAllDirectoryFilter implements DirectoryFilter {
- /** Singleton instance. */
- public static final AcceptAllDirectoryFilter INSTANCE = new AcceptAllDirectoryFilter();
-
- /** {@inheritDoc} */
- @Override public boolean test(String name) {
- return true;
- }
- }
-
- /**
- * Filter which uses prefix to filter files.
- */
- static class PrefixDirectoryFilter implements DirectoryFilter {
- /** Prefix. */
- private final String prefix;
-
- /**
- * Constructor.
- *
- * @param prefix Prefix.
- */
- public PrefixDirectoryFilter(String prefix) {
- assert prefix != null;
-
- this.prefix = normalize(prefix);
- }
-
- /** {@inheritDoc} */
- @Override public boolean test(String name) {
- return normalize(name).startsWith(prefix);
- }
- }
-
- /**
- * Filter which uses exact comparison.
- */
- static class ExactDirectoryFilter implements DirectoryFilter {
- /** Name. */
- private final String name;
-
- /**
- * Constructor.
- *
- * @param name Name.
- */
- public ExactDirectoryFilter(String name) {
- this.name = normalize(name);
- }
-
- /** {@inheritDoc} */
- @Override public boolean test(String name) {
- return normalize(name).equals(this.name);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
deleted file mode 100644
index a90007f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Simple structure to hold Hadoop directory locations.
- */
-public class HadoopLocations {
- /** Hadoop home. */
- private final String home;
-
- /** Common home. */
- private final String common;
-
- /** HDFS home. */
- private final String hdfs;
-
- /** Mapred home. */
- private final String mapred;
-
- /** Whether common home exists. */
- private final boolean commonExists;
-
- /** Whether HDFS home exists. */
- private final boolean hdfsExists;
-
- /** Whether mapred home exists. */
- private final boolean mapredExists;
-
- /**
- * Constructor.
- *
- * @param home Hadoop home.
- * @param common Common home.
- * @param hdfs HDFS home.
- * @param mapred Mapred home.
- */
- public HadoopLocations(String home, String common, String hdfs, String mapred) {
- assert common != null && hdfs != null && mapred != null;
-
- this.home = home;
- this.common = common;
- this.hdfs = hdfs;
- this.mapred = mapred;
-
- commonExists = HadoopClasspathUtils.exists(common);
- hdfsExists = HadoopClasspathUtils.exists(hdfs);
- mapredExists = HadoopClasspathUtils.exists(mapred);
- }
-
- /**
- * @return Hadoop home.
- */
- public String home() {
- return home;
- }
-
- /**
- * @return Common home.
- */
- public String common() {
- return common;
- }
-
- /**
- * @return HDFS home.
- */
- public String hdfs() {
- return hdfs;
- }
-
- /**
- * @return Mapred home.
- */
- public String mapred() {
- return mapred;
- }
-
- /**
- * @return Whether common home exists.
- */
- public boolean commonExists() {
- return commonExists;
- }
-
- /**
- * @return Whether HDFS home exists.
- */
- public boolean hdfsExists() {
- return hdfsExists;
- }
-
- /**
- * @return Whether mapred home exists.
- */
- public boolean mapredExists() {
- return mapredExists;
- }
-
- /**
- * Whether all required directories exists.
- *
- * @return {@code True} if exists.
- */
- public boolean valid() {
- return commonExists && hdfsExists && mapredExists;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index a0f30eb..595474c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -382,7 +382,7 @@ public class HadoopV2Job implements HadoopJob {
* @throws Exception On error.
*/
private void stopHadoopFsDaemons(ClassLoader ldr) throws Exception {
- Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLASS_NAME);
+ Class<?> daemonCls = ldr.loadClass(HadoopClassLoader.CLS_DAEMON);
Method m = daemonCls.getMethod("dequeueAndStopAll");
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
index 02d98d0..2fd7777 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
@@ -94,7 +94,7 @@ public class HadoopClassLoaderTest extends TestCase {
CircularWithoutHadoop.class,
};
- for (Class c: positiveClasses)
+ for (Class c : positiveClasses)
assertTrue(c.getName(), ldr.hasExternalDependencies(c.getName()));
// Negative cases:
@@ -104,7 +104,7 @@ public class HadoopClassLoaderTest extends TestCase {
Without.class,
};
- for (Class c: negativeClasses)
+ for (Class c : negativeClasses)
assertFalse(c.getName(), ldr.hasExternalDependencies(c.getName()));
}
}
\ No newline at end of file
[2/2] ignite git commit: IGNITE-3912: Preliminary changes.
Posted by vo...@apache.org.
IGNITE-3912: Preliminary changes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2fe0272b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2fe0272b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2fe0272b
Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 2fe0272ba5f2c9003ab57844ba730a384bb67ca1
Parents: 2474e2b
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Sep 16 12:55:00 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Sep 16 12:55:00 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.bat | 2 +-
bin/include/setenv.sh | 2 +-
.../processors/hadoop/HadoopClasspathMain.java | 44 ++
.../processors/hadoop/HadoopClasspathUtils.java | 461 ++++++++++++
.../processors/hadoop/HadoopLocations.java | 123 ++++
.../processors/hadoop/HadoopClassLoader.java | 697 ++-----------------
.../hadoop/HadoopClassLoaderUtils.java | 684 ++++++++++++++++++
.../processors/hadoop/HadoopClasspathMain.java | 44 --
.../processors/hadoop/HadoopClasspathUtils.java | 461 ------------
.../processors/hadoop/HadoopLocations.java | 123 ----
.../processors/hadoop/v2/HadoopV2Job.java | 2 +-
.../hadoop/HadoopClassLoaderTest.java | 4 +-
12 files changed, 1365 insertions(+), 1282 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 9d55521..2e8eac6 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -47,7 +47,7 @@ if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
-if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
+if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index a85cba3..54f5831 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -84,7 +84,7 @@ if [ "${HADOOP_EDITION}" == "1" ]; then
fi
fi
- IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \
+ IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/'*' \
org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
statusCode=${?}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
new file mode 100644
index 0000000..4069496
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Main class to compose Hadoop classpath depending on the environment.
+ * This class is designed to be independent on any Ignite classes as possible.
+ * Please make sure to pass the path separator character as the 1st parameter to the main method.
+ */
+public class HadoopClasspathMain {
+ /**
+ * Main method to be executed from scripts. It prints the classpath to the standard output.
+ *
+ * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1)
+ throw new IllegalArgumentException("Path separator must be passed as the first argument.");
+
+ String separator = args[0];
+
+ StringBuilder sb = new StringBuilder();
+
+ for (String path : HadoopClasspathUtils.classpathForProcess())
+ sb.append(path).append(separator);
+
+ System.out.println(sb);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
new file mode 100644
index 0000000..a0ef7d7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -0,0 +1,461 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Hadoop classpath utilities.
+ */
+public class HadoopClasspathUtils {
+ /** Prefix directory. */
+ public static final String PREFIX = "HADOOP_PREFIX";
+
+ /** Hadoop home directory. */
+ public static final String HOME = "HADOOP_HOME";
+
+ /** Hadoop common directory. */
+ public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
+
+ /** Hadoop HDFS directory. */
+ public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
+
+ /** Hadoop mapred directory. */
+ public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
+
+ /** Arbitrary additional dependencies. Compliant with standard Java classpath resolution. */
+ public static final String HADOOP_USER_LIBS = "HADOOP_USER_LIBS";
+
+ /** Empty string. */
+ private static final String EMPTY_STR = "";
+
+ /**
+ * Gets Hadoop class path as list of classpath elements for process.
+ *
+ * @return List of the class path elements.
+ * @throws IOException If failed.
+ */
+ public static List<String> classpathForProcess() throws IOException {
+ List<String> res = new ArrayList<>();
+
+ for (final SearchDirectory dir : classpathDirectories()) {
+ File[] files = dir.files();
+
+ if (dir.useWildcard()) {
+ if (files.length > 0)
+ res.add(dir.absolutePath() + File.separator + '*');
+ }
+ else {
+ for (File file : files)
+ res.add(file.getAbsolutePath());
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Gets Hadoop class path as a list of URLs (for in-process class loader usage).
+ *
+ * @return List of class path URLs.
+ * @throws IOException If failed.
+ */
+ public static List<URL> classpathForClassLoader() throws IOException {
+ List<URL> res = new ArrayList<>();
+
+ for (SearchDirectory dir : classpathDirectories()) {
+ for (File file : dir.files()) {
+ try {
+ res.add(file.toURI().toURL());
+ }
+ catch (MalformedURLException e) {
+ throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Gets Hadoop locations.
+ *
+ * @return The locations as determined from the environment.
+ */
+ public static HadoopLocations locations() throws IOException {
+ // Query environment.
+ String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR));
+
+ String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR);
+ String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR);
+ String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
+
+ // If any composite location is defined, use only them.
+ if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) {
+ HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+
+ if (res.valid())
+ return res;
+ else
+ throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " +
+ "either undefined or point to nonexistent directories [" +
+ "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " +
+ "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
+ "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
+ }
+ else if (!isEmpty(hadoopHome)) {
+ // All further checks will be based on HADOOP_HOME, so check for it's existence.
+ if (!exists(hadoopHome))
+ throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
+ "variable points to nonexistent directory: " + hadoopHome);
+
+ // Probe Apache Hadoop.
+ HadoopLocations res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs",
+ hadoopHome + "/share/hadoop/mapreduce"
+ );
+
+ if (res.valid())
+ return res;
+
+ // Probe CDH.
+ res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome,
+ hadoopHome + "/../hadoop-hdfs",
+ hadoopHome + "/../hadoop-mapreduce"
+ );
+
+ if (res.valid())
+ return res;
+
+ // Probe HDP.
+ res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client",
+ hadoopHome + "/../hadoop-mapreduce-client"
+ );
+
+ if (res.valid())
+ return res;
+
+ // Failed.
+ throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
+ "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome);
+ }
+ else {
+ // Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
+ throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " +
+ "variable and point it to your Hadoop distribution).");
+ }
+ }
+
+ /**
+ * Gets base directories to discover classpath elements in.
+ *
+ * @return Collection of directory and mask pairs.
+ * @throws IOException if a mandatory classpath location is not found.
+ */
+ private static Collection<SearchDirectory> classpathDirectories() throws IOException {
+ HadoopLocations loc = locations();
+
+ Collection<SearchDirectory> res = new ArrayList<>();
+
+ res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+ res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+ res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+
+ res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-common-")));
+ res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-auth-")));
+
+ res.add(new SearchDirectory(new File(loc.hdfs()), new PrefixDirectoryFilter("hadoop-hdfs-")));
+
+ res.add(new SearchDirectory(new File(loc.mapred()),
+ new PrefixDirectoryFilter("hadoop-mapreduce-client-common")));
+ res.add(new SearchDirectory(new File(loc.mapred()),
+ new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
+
+ res.addAll(parseUserLibs());
+
+ return res;
+ }
+
+ /**
+ * Parse user libs.
+ *
+ * @return Parsed libs search patterns.
+ * @throws IOException If failed.
+ */
+ public static Collection<SearchDirectory> parseUserLibs() throws IOException {
+ return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null));
+ }
+
+ /**
+ * Parse user libs.
+ *
+ * @param str String.
+ * @return Result.
+ * @throws IOException If failed.
+ */
+ public static Collection<SearchDirectory> parseUserLibs(String str) throws IOException {
+ Collection<SearchDirectory> res = new LinkedList<>();
+
+ if (!isEmpty(str)) {
+ String[] tokens = normalize(str).split(File.pathSeparator);
+
+ for (String token : tokens) {
+ // Skip empty tokens.
+ if (isEmpty(token))
+ continue;
+
+ File file = new File(token);
+ File dir = file.getParentFile();
+
+ if (token.endsWith("*")) {
+ assert dir != null;
+
+ res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, false));
+ }
+ else {
+ // Met "/" or "C:\" pattern, nothing to do with it.
+ if (dir == null)
+ continue;
+
+ res.add(new SearchDirectory(dir, new ExactDirectoryFilter(file.getName()), false));
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Get system property or environment variable with the given name.
+ *
+ * @param name Variable name.
+ * @param dflt Default value.
+ * @return Value.
+ */
+ private static String systemOrEnv(String name, String dflt) {
+ String res = System.getProperty(name);
+
+ if (res == null)
+ res = System.getenv(name);
+
+ return res != null ? res : dflt;
+ }
+
+ /**
+ * Answers if the given path denotes existing directory.
+ *
+ * @param path The directory path.
+ * @return {@code True} if the given path denotes an existing directory.
+ */
+ public static boolean exists(String path) {
+ if (path == null)
+ return false;
+
+ Path p = Paths.get(path);
+
+ return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+ }
+
+ /**
+ * Check if string is empty.
+ *
+ * @param val Value.
+ * @return {@code True} if empty.
+ */
+ private static boolean isEmpty(String val) {
+ return val == null || val.isEmpty();
+ }
+
+ /**
+ * NOramlize the string.
+ *
+ * @param str String.
+ * @return Normalized string.
+ */
+ private static String normalize(String str) {
+ assert str != null;
+
+ return str.trim().toLowerCase();
+ }
+
+ /**
+ * Simple pair-like structure to hold directory name and a mask assigned to it.
+ */
+ public static class SearchDirectory {
+ /** File. */
+ private final File dir;
+
+ /** Filter. */
+ private final DirectoryFilter filter;
+
+ /** Whether directory must exist. */
+ private final boolean strict;
+
+ /**
+ * Constructor for directory search with strict rule.
+ *
+ * @param dir Directory.
+ * @param filter Filter.
+ * @throws IOException If failed.
+ */
+ private SearchDirectory(File dir, DirectoryFilter filter) throws IOException {
+ this(dir, filter, true);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param dir Directory.
+ * @param filter Filter.
+ * @param strict Whether directory must exist.
+ * @throws IOException If failed.
+ */
+ private SearchDirectory(File dir, DirectoryFilter filter, boolean strict) throws IOException {
+ this.dir = dir;
+ this.filter = filter;
+ this.strict = strict;
+
+ if (strict && !exists(dir.getAbsolutePath()))
+ throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
+ }
+
+ /**
+ * @return Absolute path.
+ */
+ public String absolutePath() {
+ return dir.getAbsolutePath();
+ }
+
+ /**
+ * @return Child files.
+ */
+ public File[] files() throws IOException {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return filter.test(name);
+ }
+ });
+
+ if (files == null) {
+ if (strict)
+ throw new IOException("Failed to get directory files [dir=" + dir + ']');
+ else
+ return new File[0];
+ }
+ else
+ return files;
+ }
+
+ /**
+ * @return {@code True} if wildcard can be used.
+ */
+ public boolean useWildcard() {
+ return filter instanceof AcceptAllDirectoryFilter;
+ }
+ }
+
+ /**
+ * Directory filter interface.
+ */
+ public static interface DirectoryFilter {
+ /**
+ * Test if file with this name should be included.
+ *
+ * @param name File name.
+ * @return {@code True} if passed.
+ */
+ public boolean test(String name);
+ }
+
+ /**
+ * Filter to accept all files.
+ */
+ public static class AcceptAllDirectoryFilter implements DirectoryFilter {
+ /** Singleton instance. */
+ public static final AcceptAllDirectoryFilter INSTANCE = new AcceptAllDirectoryFilter();
+
+ /** {@inheritDoc} */
+ @Override public boolean test(String name) {
+ return true;
+ }
+ }
+
+ /**
+ * Filter which uses prefix to filter files.
+ */
+ public static class PrefixDirectoryFilter implements DirectoryFilter {
+ /** Prefix. */
+ private final String prefix;
+
+ /**
+ * Constructor.
+ *
+ * @param prefix Prefix.
+ */
+ public PrefixDirectoryFilter(String prefix) {
+ assert prefix != null;
+
+ this.prefix = normalize(prefix);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(String name) {
+ return normalize(name).startsWith(prefix);
+ }
+ }
+
+ /**
+ * Filter which uses exact comparison.
+ */
+ public static class ExactDirectoryFilter implements DirectoryFilter {
+ /** Name. */
+ private final String name;
+
+ /**
+ * Constructor.
+ *
+ * @param name Name.
+ */
+ public ExactDirectoryFilter(String name) {
+ this.name = normalize(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(String name) {
+ return normalize(name).equals(this.name);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
new file mode 100644
index 0000000..a90007f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Simple structure to hold Hadoop directory locations.
+ */
+public class HadoopLocations {
+ /** Hadoop home. */
+ private final String home;
+
+ /** Common home. */
+ private final String common;
+
+ /** HDFS home. */
+ private final String hdfs;
+
+ /** Mapred home. */
+ private final String mapred;
+
+ /** Whether common home exists. */
+ private final boolean commonExists;
+
+ /** Whether HDFS home exists. */
+ private final boolean hdfsExists;
+
+ /** Whether mapred home exists. */
+ private final boolean mapredExists;
+
+ /**
+ * Constructor.
+ *
+ * @param home Hadoop home.
+ * @param common Common home.
+ * @param hdfs HDFS home.
+ * @param mapred Mapred home.
+ */
+ public HadoopLocations(String home, String common, String hdfs, String mapred) {
+ assert common != null && hdfs != null && mapred != null;
+
+ this.home = home;
+ this.common = common;
+ this.hdfs = hdfs;
+ this.mapred = mapred;
+
+ commonExists = HadoopClasspathUtils.exists(common);
+ hdfsExists = HadoopClasspathUtils.exists(hdfs);
+ mapredExists = HadoopClasspathUtils.exists(mapred);
+ }
+
+ /**
+ * @return Hadoop home.
+ */
+ public String home() {
+ return home;
+ }
+
+ /**
+ * @return Common home.
+ */
+ public String common() {
+ return common;
+ }
+
+ /**
+ * @return HDFS home.
+ */
+ public String hdfs() {
+ return hdfs;
+ }
+
+ /**
+ * @return Mapred home.
+ */
+ public String mapred() {
+ return mapred;
+ }
+
+ /**
+ * @return Whether common home exists.
+ */
+ public boolean commonExists() {
+ return commonExists;
+ }
+
+ /**
+ * @return Whether HDFS home exists.
+ */
+ public boolean hdfsExists() {
+ return hdfsExists;
+ }
+
+ /**
+ * @return Whether mapred home exists.
+ */
+ public boolean mapredExists() {
+ return mapredExists;
+ }
+
+ /**
+ * Whether all required directories exists.
+ *
+ * @return {@code True} if exists.
+ */
+ public boolean valid() {
+ return commonExists && hdfsExists && mapredExists;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 2e0e271..30a6e72 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -17,44 +17,26 @@
package org.apache.ignite.internal.processors.hadoop;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.ClassCache;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentHashMap8;
+
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.hadoop.util.NativeCodeLoader;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon;
-import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager;
-import org.apache.ignite.internal.util.ClassCache;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
-import org.jsr166.ConcurrentHashMap8;
-import org.objectweb.asm.AnnotationVisitor;
-import org.objectweb.asm.Attribute;
-import org.objectweb.asm.ClassReader;
-import org.objectweb.asm.ClassVisitor;
-import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.FieldVisitor;
-import org.objectweb.asm.Handle;
-import org.objectweb.asm.Label;
-import org.objectweb.asm.MethodVisitor;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
-import org.objectweb.asm.commons.Remapper;
-import org.objectweb.asm.commons.RemappingClassAdapter;
-
/**
* Class loader allowing explicitly load classes without delegation to parent class loader.
* Also supports class parsing for finding dependencies which contain transitive dependencies
@@ -66,8 +48,21 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
registerAsParallelCapable();
}
- /** Name of the Hadoop daemon class. */
- public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon";
+ /** Hadoop class name: Daemon. */
+ public static final String CLS_DAEMON = "org.apache.hadoop.util.Daemon";
+
+ /** Hadoop class name: ShutdownHookManager. */
+ public static final String CLS_SHUTDOWN_HOOK_MANAGER = "org.apache.hadoop.util.ShutdownHookManager";
+
+ /** Hadoop class name: NativeCodeLoader. */
+ public static final String CLS_NATIVE_CODE_LOADER = "org.apache.hadoop.util.NativeCodeLoader";
+
+ /** Hadoop class name: Daemon replacement. */
+ public static final String CLS_DAEMON_REPLACE = "org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon";
+
+ /** Hadoop class name: ShutdownHookManager replacement. */
+ public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE =
+ "org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager";
/** Name of libhadoop library. */
private static final String LIBHADOOP = "hadoop.";
@@ -82,9 +77,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
private static volatile Collection<URL> hadoopJars;
/** */
- private static final Map<String, Boolean> cache = new ConcurrentHashMap8<>();
-
- /** */
private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>();
/** Class cache. */
@@ -152,10 +144,12 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
private void initializeNativeLibraries() {
try {
// This must trigger native library load.
- Class.forName(NativeCodeLoader.class.getName(), true, APP_CLS_LDR);
+ // TODO: Do not delegate to APP LDR
+ Class.forName(CLS_NATIVE_CODE_LOADER, true, APP_CLS_LDR);
final Vector<Object> curVector = U.field(this, "nativeLibraries");
+ // TODO: Do not delegate to APP LDR
ClassLoader ldr = APP_CLS_LDR;
while (ldr != null) {
@@ -192,55 +186,24 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
}
- /**
- * Need to parse only Ignite Hadoop and IGFS classes.
- *
- * @param cls Class name.
- * @return {@code true} if we need to check this class.
- */
- private static boolean isHadoopIgfs(String cls) {
- String ignitePkgPrefix = "org.apache.ignite";
-
- int len = ignitePkgPrefix.length();
-
- return cls.startsWith(ignitePkgPrefix) && (
- cls.indexOf("igfs.", len) != -1 ||
- cls.indexOf(".fs.", len) != -1 ||
- cls.indexOf("hadoop.", len) != -1);
- }
-
- /**
- * @param cls Class name.
- * @return {@code true} If this is Hadoop class.
- */
- private static boolean isHadoop(String cls) {
- return cls.startsWith("org.apache.hadoop.");
- }
-
/** {@inheritDoc} */
@Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
try {
- if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
- if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks.
- return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
- else if (name.equals(HADOOP_DAEMON_CLASS_NAME))
+ // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
+ if (HadoopClassLoaderUtils.isHadoop(name)) {
+ if (name.equals(CLS_SHUTDOWN_HOOK_MANAGER)) // Dirty hack to get rid of Hadoop shutdown hooks.
+ return loadReplace(name, CLS_SHUTDOWN_HOOK_MANAGER_REPLACE);
+ else if (name.equals(CLS_DAEMON))
// We replace this in order to be able to forcibly stop some daemon threads
// that otherwise never stop (e.g. PeerCache runnables):
- return loadFromBytes(name, HadoopDaemon.class.getName());
+ return loadReplace(name, CLS_DAEMON_REPLACE);
return loadClassExplicitly(name, resolve);
}
- if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
- Boolean hasDeps = cache.get(name);
-
- if (hasDeps == null) {
- hasDeps = hasExternalDependencies(name);
-
- cache.put(name, hasDeps);
- }
-
- if (hasDeps)
+ // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
+ if (HadoopClassLoaderUtils.isHadoopIgfs(name)) {
+ if (hasExternalDependencies(name))
return loadClassExplicitly(name, resolve);
}
@@ -252,55 +215,31 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * @param name Name.
- * @param replace Replacement.
+ * Load a class replacing it with our own implementation.
+ *
+ * @param originalName Name.
+ * @param replaceName Replacement.
* @return Class.
*/
- private Class<?> loadFromBytes(final String name, final String replace) {
- synchronized (getClassLoadingLock(name)) {
+ private Class<?> loadReplace(final String originalName, final String replaceName) {
+ synchronized (getClassLoadingLock(originalName)) {
// First, check if the class has already been loaded
- Class c = findLoadedClass(name);
+ Class c = findLoadedClass(originalName);
if (c != null)
return c;
- byte[] bytes = bytesCache.get(name);
+ byte[] bytes = bytesCache.get(originalName);
if (bytes == null) {
- InputStream in = loadClassBytes(getParent(), replace);
-
- ClassReader rdr;
-
- try {
- rdr = new ClassReader(in);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- ClassWriter w = new ClassWriter(Opcodes.ASM4);
-
- rdr.accept(new RemappingClassAdapter(w, new Remapper() {
- /** */
- String replaceType = replace.replace('.', '/');
+ InputStream in = HadoopClassLoaderUtils.loadClassBytes(getParent(), replaceName);
- /** */
- String nameType = name.replace('.', '/');
+ bytes = HadoopClassLoaderUtils.loadReplace(in, originalName, replaceName);
- @Override public String map(String type) {
- if (type.equals(replaceType))
- return nameType;
-
- return type;
- }
- }), ClassReader.EXPAND_FRAMES);
-
- bytes = w.toByteArray();
-
- bytesCache.put(name, bytes);
+ bytesCache.put(originalName, bytes);
}
- return defineClass(name, bytes, 0, bytes.length);
+ return defineClass(originalName, bytes, 0, bytes.length);
}
}
@@ -347,111 +286,13 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * @param ldr Loader.
- * @param clsName Class.
- * @return Input stream.
- */
- @Nullable private InputStream loadClassBytes(ClassLoader ldr, String clsName) {
- return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
- }
-
- /**
* Check whether class has external dependencies on Hadoop.
*
* @param clsName Class name.
* @return {@code True} if class has external dependencies.
*/
boolean hasExternalDependencies(String clsName) {
- CollectingContext ctx = new CollectingContext();
-
- ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
- ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
- ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
- ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor);
-
- return hasExternalDependencies(clsName, ctx);
- }
-
- /**
- * Check whether class has external dependencies on Hadoop.
- *
- * @param clsName Class name.
- * @param ctx Context.
- * @return {@code true} If the class has external dependencies.
- */
- boolean hasExternalDependencies(String clsName, CollectingContext ctx) {
- if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
- return true;
-
- // Try to get from parent to check if the type accessible.
- InputStream in = loadClassBytes(getParent(), clsName);
-
- if (in == null) // The class is external itself, it must be loaded from this class loader.
- return true;
-
- if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies.
- return false;
-
- final ClassReader rdr;
-
- try {
- rdr = new ClassReader(in);
- }
- catch (IOException e) {
- throw new RuntimeException("Failed to read class: " + clsName, e);
- }
-
- ctx.visited.add(clsName);
-
- rdr.accept(ctx.clsVisitor, 0);
-
- if (ctx.found) // We already know that we have dependencies, no need to check parent.
- return true;
-
- // Here we are known to not have any dependencies but possibly we have a parent which has them.
- int idx = clsName.lastIndexOf('$');
-
- if (idx == -1) // No parent class.
- return false;
-
- String parentCls = clsName.substring(0, idx);
-
- if (ctx.visited.contains(parentCls))
- return false;
-
- Boolean res = cache.get(parentCls);
-
- if (res == null)
- res = hasExternalDependencies(parentCls, ctx);
-
- return res;
- }
-
- /**
- * @param name Class name.
- * @return {@code true} If this is a valid class name.
- */
- private static boolean validateClassName(String name) {
- int len = name.length();
-
- if (len <= 1)
- return false;
-
- if (!Character.isJavaIdentifierStart(name.charAt(0)))
- return false;
-
- boolean hasDot = false;
-
- for (int i = 1; i < len; i++) {
- char c = name.charAt(i);
-
- if (c == '.')
- hasDot = true;
- else if (!Character.isJavaIdentifierPart(c))
- return false;
- }
-
- return hasDot;
+ return HadoopClassLoaderUtils.hasExternalDependencies(clsName, getParent());
}
/**
@@ -519,446 +360,4 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
public String name() {
return name;
}
-
- /**
- * Context for dependencies collection.
- */
- private class CollectingContext {
- /** Visited classes. */
- private final Set<String> visited = new HashSet<>();
-
- /** Whether dependency found. */
- private boolean found;
-
- /** Annotation visitor. */
- private AnnotationVisitor annVisitor;
-
- /** Method visitor. */
- private MethodVisitor mthdVisitor;
-
- /** Field visitor. */
- private FieldVisitor fldVisitor;
-
- /** Class visitor. */
- private ClassVisitor clsVisitor;
-
- /**
- * Processes a method descriptor
- * @param methDesc The method desc String.
- */
- void onMethodsDesc(final String methDesc) {
- // Process method return type:
- onType(Type.getReturnType(methDesc));
-
- if (found)
- return;
-
- // Process method argument types:
- for (Type t: Type.getArgumentTypes(methDesc)) {
- onType(t);
-
- if (found)
- return;
- }
- }
-
- /**
- * Processes dependencies of a class.
- *
- * @param depCls The class name as dot-notated FQN.
- */
- void onClass(final String depCls) {
- assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation.
- assert depCls.charAt(0) != 'L' : depCls;
- assert validateClassName(depCls) : depCls;
-
- if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes.
- return;
-
- if (visited.contains(depCls))
- return;
-
- Boolean res = cache.get(depCls);
-
- if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, this)))
- found = true;
- }
-
- /**
- * Analyses dependencies of given type.
- *
- * @param t The type to process.
- */
- void onType(Type t) {
- if (t == null)
- return;
-
- int sort = t.getSort();
-
- switch (sort) {
- case Type.ARRAY:
- onType(t.getElementType());
-
- break;
-
- case Type.OBJECT:
- onClass(t.getClassName());
-
- break;
- }
- }
-
- /**
- * Analyses dependencies of given object type.
- *
- * @param objType The object type to process.
- */
- void onInternalTypeName(String objType) {
- if (objType == null)
- return;
-
- assert objType.length() > 1 : objType;
-
- if (objType.charAt(0) == '[')
- // handle array. In this case this is a type descriptor notation, like "[Ljava/lang/Object;"
- onType(objType);
- else {
- assert objType.indexOf('.') == -1 : objType; // Must be slash-separated FQN.
-
- String clsName = objType.replace('/', '.'); // Convert it to dot notation.
-
- onClass(clsName); // Process.
- }
- }
-
- /**
- * Type description analyser.
- *
- * @param desc The description.
- */
- void onType(String desc) {
- if (!F.isEmpty(desc)) {
- if (desc.length() <= 1)
- return; // Optimization: filter out primitive types in early stage.
-
- Type t = Type.getType(desc);
-
- onType(t);
- }
- }
- }
-
- /**
- * Annotation visitor.
- */
- private static class CollectingAnnotationVisitor extends AnnotationVisitor {
- /** */
- final CollectingContext ctx;
-
- /**
- * Annotation visitor.
- *
- * @param ctx The collector.
- */
- CollectingAnnotationVisitor(CollectingContext ctx) {
- super(Opcodes.ASM4);
-
- this.ctx = ctx;
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitAnnotation(String name, String desc) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public void visitEnum(String name, String desc, String val) {
- if (ctx.found)
- return;
-
- ctx.onType(desc);
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitArray(String name) {
- return ctx.found ? null : this;
- }
-
- /** {@inheritDoc} */
- @Override public void visit(String name, Object val) {
- if (ctx.found)
- return;
-
- if (val instanceof Type)
- ctx.onType((Type)val);
- }
-
- /** {@inheritDoc} */
- @Override public void visitEnd() {
- // No-op.
- }
- }
-
- /**
- * Field visitor.
- */
- private static class CollectingFieldVisitor extends FieldVisitor {
- /** Collector. */
- private final CollectingContext ctx;
-
- /** Annotation visitor. */
- private final AnnotationVisitor av;
-
- /**
- * Constructor.
- */
- CollectingFieldVisitor(CollectingContext ctx, AnnotationVisitor av) {
- super(Opcodes.ASM4);
-
- this.ctx = ctx;
- this.av = av;
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return ctx.found ? null : av;
- }
-
- /** {@inheritDoc} */
- @Override public void visitAttribute(Attribute attr) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void visitEnd() {
- // No-op.
- }
- }
-
- /**
- * Class visitor.
- */
- private static class CollectingClassVisitor extends ClassVisitor {
- /** Collector. */
- private final CollectingContext ctx;
-
- /** Annotation visitor. */
- private final AnnotationVisitor av;
-
- /** Method visitor. */
- private final MethodVisitor mv;
-
- /** Field visitor. */
- private final FieldVisitor fv;
-
- /**
- * Constructor.
- *
- * @param ctx Collector.
- * @param av Annotation visitor.
- * @param mv Method visitor.
- * @param fv Field visitor.
- */
- CollectingClassVisitor(CollectingContext ctx, AnnotationVisitor av, MethodVisitor mv, FieldVisitor fv) {
- super(Opcodes.ASM4);
-
- this.ctx = ctx;
- this.av = av;
- this.mv = mv;
- this.fv = fv;
- }
-
- /** {@inheritDoc} */
- @Override public void visit(int i, int i2, String name, String signature, String superName, String[] ifaces) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(superName);
-
- if (ctx.found)
- return;
-
- if (ifaces != null) {
- for (String iface : ifaces) {
- ctx.onInternalTypeName(iface);
-
- if (ctx.found)
- return;
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return ctx.found ? null : av;
- }
-
- /** {@inheritDoc} */
- @Override public void visitInnerClass(String name, String outerName, String innerName, int i) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(name);
- }
-
- /** {@inheritDoc} */
- @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return ctx.found ? null : fv;
- }
-
- /** {@inheritDoc} */
- @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature,
- String[] exceptions) {
- if (ctx.found)
- return null;
-
- ctx.onMethodsDesc(desc);
-
- // Process declared method exceptions:
- if (exceptions != null) {
- for (String e : exceptions)
- ctx.onInternalTypeName(e);
- }
-
- return ctx.found ? null : mv;
- }
- }
-
- /**
- * Method visitor.
- */
- private static class CollectingMethodVisitor extends MethodVisitor {
- /** Collector. */
- private final CollectingContext ctx;
-
- /** Annotation visitor. */
- private final AnnotationVisitor av;
-
- /**
- * Constructor.
- *
- * @param ctx Collector.
- * @param av Annotation visitor.
- */
- private CollectingMethodVisitor(CollectingContext ctx, AnnotationVisitor av) {
- super(Opcodes.ASM4);
-
- this.ctx = ctx;
- this.av = av;
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return ctx.found ? null : av;
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) {
- if (ctx.found)
- return null;
-
- ctx.onType(desc);
-
- return ctx.found ? null : av;
- }
-
- /** {@inheritDoc} */
- @Override public AnnotationVisitor visitAnnotationDefault() {
- return ctx.found ? null : av;
- }
-
- /** {@inheritDoc} */
- @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(owner);
-
- if (ctx.found)
- return;
-
- ctx.onType(desc);
- }
-
- /** {@inheritDoc} */
- @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, Object... bsmArgs) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void visitFrame(int type, int nLoc, Object[] locTypes, int nStack, Object[] stackTypes) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void visitLocalVariable(String name, String desc, String signature, Label lb,
- Label lb2, int i) {
- if (ctx.found)
- return;
-
- ctx.onType(desc);
- }
-
- /** {@inheritDoc} */
- @Override public void visitMethodInsn(int i, String owner, String name, String desc) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(owner);
-
- if (ctx.found)
- return;
-
- ctx.onMethodsDesc(desc);
- }
-
- /** {@inheritDoc} */
- @Override public void visitMultiANewArrayInsn(String desc, int dim) {
- if (ctx.found)
- return;
-
- ctx.onType(desc);
- }
-
- /** {@inheritDoc} */
- @Override public void visitTryCatchBlock(Label start, Label end, Label hndl, String typeStr) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(typeStr);
- }
-
- /** {@inheritDoc} */
- @Override public void visitTypeInsn(int opcode, String type) {
- if (ctx.found)
- return;
-
- ctx.onInternalTypeName(type);
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
new file mode 100644
index 0000000..3415d6a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
@@ -0,0 +1,684 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentHashMap8;
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.Attribute;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.FieldVisitor;
+import org.objectweb.asm.Handle;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Remapper;
+import org.objectweb.asm.commons.RemappingClassAdapter;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Utility methods for Hadoop classloader required to avoid direct 3rd-party dependencies in class loader.
+ */
+public class HadoopClassLoaderUtils {
+ /** Cache for resolved dependency info. */
+ private static final Map<String, Boolean> dependenciesCache = new ConcurrentHashMap8<>();
+
+ /**
+ * Load special replacement and impersonate
+ *
+ * @param in Input stream.
+ * @param originalName Original class name.
+ * @param replaceName Replacer class name.
+ * @return Result.
+ */
+ public static byte[] loadReplace(InputStream in, final String originalName, final String replaceName) {
+ ClassReader rdr;
+
+ try {
+ rdr = new ClassReader(in);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ ClassWriter w = new ClassWriter(Opcodes.ASM4);
+
+ rdr.accept(new RemappingClassAdapter(w, new Remapper() {
+ /** */
+ String replaceType = replaceName.replace('.', '/');
+
+ /** */
+ String nameType = originalName.replace('.', '/');
+
+ @Override public String map(String type) {
+ if (type.equals(replaceType))
+ return nameType;
+
+ return type;
+ }
+ }), ClassReader.EXPAND_FRAMES);
+
+ return w.toByteArray();
+ }
+
+ /**
+ * @param cls Class name.
+ * @return {@code true} If this is Hadoop class.
+ */
+ public static boolean isHadoop(String cls) {
+ return cls.startsWith("org.apache.hadoop.");
+ }
+
+ /**
+ * Need to parse only Ignite Hadoop and IGFS classes.
+ *
+ * @param cls Class name.
+ * @return {@code true} if we need to check this class.
+ */
+ public static boolean isHadoopIgfs(String cls) {
+ String ignitePkgPrefix = "org.apache.ignite";
+
+ int len = ignitePkgPrefix.length();
+
+ return cls.startsWith(ignitePkgPrefix) && (
+ cls.indexOf("igfs.", len) != -1 ||
+ cls.indexOf(".fs.", len) != -1 ||
+ cls.indexOf("hadoop.", len) != -1);
+ }
+
+ /**
+ * @param ldr Loader.
+ * @param clsName Class.
+ * @return Input stream.
+ */
+ @Nullable public static InputStream loadClassBytes(ClassLoader ldr, String clsName) {
+ return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
+ }
+
+ /**
+ * Check whether class has external dependencies on Hadoop.
+ *
+ * @param clsName Class name.
+ * @param parentClsLdr Parent class loader.
+ * @return {@code True} if class has external dependencies.
+ */
+ static boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr) {
+ Boolean hasDeps = dependenciesCache.get(clsName);
+
+ if (hasDeps == null) {
+ CollectingContext ctx = new CollectingContext(parentClsLdr);
+
+ ctx.annVisitor = new CollectingAnnotationVisitor(ctx);
+ ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor);
+ ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor);
+ ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor);
+
+ hasDeps = hasExternalDependencies(clsName, parentClsLdr, ctx);
+
+ dependenciesCache.put(clsName, hasDeps);
+ }
+
+ return hasDeps;
+ }
+
+ /**
+ * Check whether class has external dependencies on Hadoop.
+ *
+ * @param clsName Class name.
+ * @param parentClsLdr Parent class loader.
+ * @param ctx Context.
+ * @return {@code true} If the class has external dependencies.
+ */
+ static boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr, CollectingContext ctx) {
+ if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
+ return true;
+
+ // Try to get from parent to check if the type accessible.
+ InputStream in = loadClassBytes(parentClsLdr, clsName);
+
+ if (in == null) // The class is external itself, it must be loaded from this class loader.
+ return true;
+
+ if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies.
+ return false;
+
+ final ClassReader rdr;
+
+ try {
+ rdr = new ClassReader(in);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed to read class: " + clsName, e);
+ }
+
+ ctx.visited.add(clsName);
+
+ rdr.accept(ctx.clsVisitor, 0);
+
+ if (ctx.found) // We already know that we have dependencies, no need to check parent.
+ return true;
+
+ // Here we are known to not have any dependencies but possibly we have a parent which has them.
+ int idx = clsName.lastIndexOf('$');
+
+ if (idx == -1) // No parent class.
+ return false;
+
+ String parentCls = clsName.substring(0, idx);
+
+ if (ctx.visited.contains(parentCls))
+ return false;
+
+ Boolean res = dependenciesCache.get(parentCls);
+
+ if (res == null)
+ res = hasExternalDependencies(parentCls, parentClsLdr, ctx);
+
+ return res;
+ }
+
+ /**
+ * @param name Class name.
+ * @return {@code true} If this is a valid class name.
+ */
+ private static boolean validateClassName(String name) {
+ int len = name.length();
+
+ if (len <= 1)
+ return false;
+
+ if (!Character.isJavaIdentifierStart(name.charAt(0)))
+ return false;
+
+ boolean hasDot = false;
+
+ for (int i = 1; i < len; i++) {
+ char c = name.charAt(i);
+
+ if (c == '.')
+ hasDot = true;
+ else if (!Character.isJavaIdentifierPart(c))
+ return false;
+ }
+
+ return hasDot;
+ }
+
+ /**
+ * Context for dependencies collection.
+ */
+ private static class CollectingContext {
+ /** Visited classes. */
+ private final Set<String> visited = new HashSet<>();
+
+ /** Parent class loader. */
+ private final ClassLoader parentClsLdr;
+
+ /** Whether dependency found. */
+ private boolean found;
+
+ /** Annotation visitor. */
+ private AnnotationVisitor annVisitor;
+
+ /** Method visitor. */
+ private MethodVisitor mthdVisitor;
+
+ /** Field visitor. */
+ private FieldVisitor fldVisitor;
+
+ /** Class visitor. */
+ private ClassVisitor clsVisitor;
+
+ /**
+ * Constrcutor.
+ *
+ * @param parentClsLdr Parent class loader.
+ */
+ private CollectingContext(ClassLoader parentClsLdr) {
+ this.parentClsLdr = parentClsLdr;
+ }
+
+ /**
+ * Processes a method descriptor
+ * @param methDesc The method desc String.
+ */
+ void onMethodsDesc(final String methDesc) {
+ // Process method return type:
+ onType(Type.getReturnType(methDesc));
+
+ if (found)
+ return;
+
+ // Process method argument types:
+ for (Type t: Type.getArgumentTypes(methDesc)) {
+ onType(t);
+
+ if (found)
+ return;
+ }
+ }
+
+ /**
+ * Processes dependencies of a class.
+ *
+ * @param depCls The class name as dot-notated FQN.
+ */
+ void onClass(final String depCls) {
+ assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation.
+ assert depCls.charAt(0) != 'L' : depCls;
+ assert validateClassName(depCls) : depCls;
+
+ if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes.
+ return;
+
+ if (visited.contains(depCls))
+ return;
+
+ Boolean res = dependenciesCache.get(depCls);
+
+ if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, parentClsLdr, this)))
+ found = true;
+ }
+
+ /**
+ * Analyses dependencies of given type.
+ *
+ * @param t The type to process.
+ */
+ void onType(Type t) {
+ if (t == null)
+ return;
+
+ int sort = t.getSort();
+
+ switch (sort) {
+ case Type.ARRAY:
+ onType(t.getElementType());
+
+ break;
+
+ case Type.OBJECT:
+ onClass(t.getClassName());
+
+ break;
+ }
+ }
+
+ /**
+ * Analyses dependencies of given object type.
+ *
+ * @param objType The object type to process.
+ */
+ void onInternalTypeName(String objType) {
+ if (objType == null)
+ return;
+
+ assert objType.length() > 1 : objType;
+
+ if (objType.charAt(0) == '[')
+ // handle array. In this case this is a type descriptor notation, like "[Ljava/lang/Object;"
+ onType(objType);
+ else {
+ assert objType.indexOf('.') == -1 : objType; // Must be slash-separated FQN.
+
+ String clsName = objType.replace('/', '.'); // Convert it to dot notation.
+
+ onClass(clsName); // Process.
+ }
+ }
+
+ /**
+ * Type description analyser.
+ *
+ * @param desc The description.
+ */
+ void onType(String desc) {
+ if (!F.isEmpty(desc)) {
+ if (desc.length() <= 1)
+ return; // Optimization: filter out primitive types in early stage.
+
+ Type t = Type.getType(desc);
+
+ onType(t);
+ }
+ }
+ }
+
+ /**
+ * Annotation visitor.
+ */
+ private static class CollectingAnnotationVisitor extends AnnotationVisitor {
+ /** */
+ final CollectingContext ctx;
+
+ /**
+ * Annotation visitor.
+ *
+ * @param ctx The collector.
+ */
+ CollectingAnnotationVisitor(CollectingContext ctx) {
+ super(Opcodes.ASM4);
+
+ this.ctx = ctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitAnnotation(String name, String desc) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitEnum(String name, String desc, String val) {
+ if (ctx.found)
+ return;
+
+ ctx.onType(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitArray(String name) {
+ return ctx.found ? null : this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visit(String name, Object val) {
+ if (ctx.found)
+ return;
+
+ if (val instanceof Type)
+ ctx.onType((Type)val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitEnd() {
+ // No-op.
+ }
+ }
+
+ /**
+ * Field visitor.
+ */
+ private static class CollectingFieldVisitor extends FieldVisitor {
+ /** Collector. */
+ private final CollectingContext ctx;
+
+ /** Annotation visitor. */
+ private final AnnotationVisitor av;
+
+ /**
+ * Constructor.
+ */
+ CollectingFieldVisitor(CollectingContext ctx, AnnotationVisitor av) {
+ super(Opcodes.ASM4);
+
+ this.ctx = ctx;
+ this.av = av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return ctx.found ? null : av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitAttribute(Attribute attr) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitEnd() {
+ // No-op.
+ }
+ }
+
+ /**
+ * Class visitor.
+ */
+ private static class CollectingClassVisitor extends ClassVisitor {
+ /** Collector. */
+ private final CollectingContext ctx;
+
+ /** Annotation visitor. */
+ private final AnnotationVisitor av;
+
+ /** Method visitor. */
+ private final MethodVisitor mv;
+
+ /** Field visitor. */
+ private final FieldVisitor fv;
+
+ /**
+ * Constructor.
+ *
+ * @param ctx Collector.
+ * @param av Annotation visitor.
+ * @param mv Method visitor.
+ * @param fv Field visitor.
+ */
+ CollectingClassVisitor(CollectingContext ctx, AnnotationVisitor av, MethodVisitor mv, FieldVisitor fv) {
+ super(Opcodes.ASM4);
+
+ this.ctx = ctx;
+ this.av = av;
+ this.mv = mv;
+ this.fv = fv;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visit(int i, int i2, String name, String signature, String superName, String[] ifaces) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(superName);
+
+ if (ctx.found)
+ return;
+
+ if (ifaces != null) {
+ for (String iface : ifaces) {
+ ctx.onInternalTypeName(iface);
+
+ if (ctx.found)
+ return;
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return ctx.found ? null : av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitInnerClass(String name, String outerName, String innerName, int i) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return ctx.found ? null : fv;
+ }
+
+ /** {@inheritDoc} */
+ @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature,
+ String[] exceptions) {
+ if (ctx.found)
+ return null;
+
+ ctx.onMethodsDesc(desc);
+
+ // Process declared method exceptions:
+ if (exceptions != null) {
+ for (String e : exceptions)
+ ctx.onInternalTypeName(e);
+ }
+
+ return ctx.found ? null : mv;
+ }
+ }
+
+ /**
+ * Method visitor.
+ */
+ private static class CollectingMethodVisitor extends MethodVisitor {
+ /** Collector. */
+ private final CollectingContext ctx;
+
+ /** Annotation visitor. */
+ private final AnnotationVisitor av;
+
+ /**
+ * Constructor.
+ *
+ * @param ctx Collector.
+ * @param av Annotation visitor.
+ */
+ private CollectingMethodVisitor(CollectingContext ctx, AnnotationVisitor av) {
+ super(Opcodes.ASM4);
+
+ this.ctx = ctx;
+ this.av = av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return ctx.found ? null : av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) {
+ if (ctx.found)
+ return null;
+
+ ctx.onType(desc);
+
+ return ctx.found ? null : av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AnnotationVisitor visitAnnotationDefault() {
+ return ctx.found ? null : av;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(owner);
+
+ if (ctx.found)
+ return;
+
+ ctx.onType(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, Object... bsmArgs) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitFrame(int type, int nLoc, Object[] locTypes, int nStack, Object[] stackTypes) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitLocalVariable(String name, String desc, String signature, Label lb,
+ Label lb2, int i) {
+ if (ctx.found)
+ return;
+
+ ctx.onType(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitMethodInsn(int i, String owner, String name, String desc) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(owner);
+
+ if (ctx.found)
+ return;
+
+ ctx.onMethodsDesc(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitMultiANewArrayInsn(String desc, int dim) {
+ if (ctx.found)
+ return;
+
+ ctx.onType(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitTryCatchBlock(Label start, Label end, Label hndl, String typeStr) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(typeStr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void visitTypeInsn(int opcode, String type) {
+ if (ctx.found)
+ return;
+
+ ctx.onInternalTypeName(type);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
deleted file mode 100644
index 4069496..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Main class to compose Hadoop classpath depending on the environment.
- * This class is designed to be independent on any Ignite classes as possible.
- * Please make sure to pass the path separator character as the 1st parameter to the main method.
- */
-public class HadoopClasspathMain {
- /**
- * Main method to be executed from scripts. It prints the classpath to the standard output.
- *
- * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
- */
- public static void main(String[] args) throws Exception {
- if (args.length < 1)
- throw new IllegalArgumentException("Path separator must be passed as the first argument.");
-
- String separator = args[0];
-
- StringBuilder sb = new StringBuilder();
-
- for (String path : HadoopClasspathUtils.classpathForProcess())
- sb.append(path).append(separator);
-
- System.out.println(sb);
- }
-}