You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/22 13:24:22 UTC
ignite git commit: IGNITE-3943: Hadoop: fixed scrips to match new
class loader architecture. This closes #1098.
Repository: ignite
Updated Branches:
refs/heads/ignite-1.6.8-hadoop 778878a03 -> 87981a9e2
IGNITE-3943: Hadoop: fixed scrips to match new class loader architecture. This closes #1098.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/87981a9e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/87981a9e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/87981a9e
Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 87981a9e247577b1c63768e216ca04383f4bbb33
Parents: 778878a
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Sep 22 15:40:11 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Sep 22 15:40:11 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.bat | 11 +---
bin/include/setenv.sh | 35 ++-----------
.../processors/hadoop/HadoopClassLoader.java | 6 ++-
.../processors/hadoop/HadoopClasspathMain.java | 44 ----------------
.../processors/hadoop/HadoopClasspathUtils.java | 54 ++++++--------------
.../hadoop/impl/HadoopCommandLineTest.java | 3 +-
6 files changed, 27 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 2e8eac6..6b9abf5 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -41,18 +41,11 @@ goto :eof
:: The following libraries are required for Ignite.
set IGNITE_LIBS=%IGNITE_HOME%\libs\*
-for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == "%IGNITE_HOME%\libs\optional" call :concat %%F\*
-
-if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
+for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == "%IGNITE_HOME%\libs\optional" ^
+ and not "%%F" == "%IGNITE_HOME%\libs\ignite-hadoop-impl" call :concat %%F\*
if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
-if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
-
-if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
-
-set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
-
goto :eof
:concat
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 54f5831..c6004e5 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -58,42 +58,13 @@ IGNITE_LIBS="${IGNITE_HOME}/libs/*"
for file in ${IGNITE_HOME}/libs/*
do
- if [ -d ${file} ] && [ "${file}" != "${IGNITE_HOME}"/libs/optional ]; then
+ if [ -d ${file} ] \
+ && [ "${file}" != "${IGNITE_HOME}"/libs/optional ] \
+ && [ "${file}" != "${IGNITE_HOME}"/libs/ignite-hadoop-impl ]; then
IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/*
fi
-
- if [ -d ${file} ] && [ "${file}" == "${IGNITE_HOME}"/libs/ignite-hadoop ]; then
- HADOOP_EDITION=1
- fi
done
if [ "${USER_LIBS}" != "" ]; then
IGNITE_LIBS=${USER_LIBS}${SEP}${IGNITE_LIBS}
fi
-
-if [ "${HADOOP_EDITION}" == "1" ]; then
- # Resolve constants.
- HADOOP_DEFAULTS="/etc/default/hadoop"
-
- #
- # Resolve the rest of Hadoop environment variables.
- #
- if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z "${HADOOP_MAPRED_HOME}" ]]; then
- if [ -f "$HADOOP_DEFAULTS" ]; then
- source "$HADOOP_DEFAULTS"
- fi
- fi
-
- IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/'*' \
- org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
-
- statusCode=${?}
-
- if [ "${statusCode}" -ne 0 ]; then
- exit ${statusCode}
- fi
-
- unset statusCode
-
- IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
-fi
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 60f5f55..4751283 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -311,7 +311,11 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
byte[] bytes = bytesCache.get(originalName);
if (bytes == null) {
- InputStream in = helper.loadClassBytes(getParent(), replaceName);
+ InputStream in = helper.loadClassBytes(this, replaceName);
+
+ if (in == null)
+ throw new IgniteException("Failed to replace class [originalName=" + originalName +
+ ", replaceName=" + replaceName + ']');
bytes = helper.loadReplace(in, originalName, replaceName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
deleted file mode 100644
index 4069496..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Main class to compose Hadoop classpath depending on the environment.
- * This class is designed to be independent on any Ignite classes as possible.
- * Please make sure to pass the path separator character as the 1st parameter to the main method.
- */
-public class HadoopClasspathMain {
- /**
- * Main method to be executed from scripts. It prints the classpath to the standard output.
- *
- * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
- */
- public static void main(String[] args) throws Exception {
- if (args.length < 1)
- throw new IllegalArgumentException("Path separator must be passed as the first argument.");
-
- String separator = args[0];
-
- StringBuilder sb = new StringBuilder();
-
- for (String path : HadoopClasspathUtils.classpathForProcess())
- sb.append(path).append(separator);
-
- System.out.println(sb);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index a0ef7d7..163ddfa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -34,6 +34,9 @@ import java.util.List;
* Hadoop classpath utilities.
*/
public class HadoopClasspathUtils {
+ /** Ignite home. */
+ private static final String IGNITE_HOME = "IGNITE_HOME";
+
/** Prefix directory. */
public static final String PREFIX = "HADOOP_PREFIX";
@@ -56,31 +59,6 @@ public class HadoopClasspathUtils {
private static final String EMPTY_STR = "";
/**
- * Gets Hadoop class path as list of classpath elements for process.
- *
- * @return List of the class path elements.
- * @throws IOException If failed.
- */
- public static List<String> classpathForProcess() throws IOException {
- List<String> res = new ArrayList<>();
-
- for (final SearchDirectory dir : classpathDirectories()) {
- File[] files = dir.files();
-
- if (dir.useWildcard()) {
- if (files.length > 0)
- res.add(dir.absolutePath() + File.separator + '*');
- }
- else {
- for (File file : files)
- res.add(file.getAbsolutePath());
- }
- }
-
- return res;
- }
-
- /**
* Gets Hadoop class path as a list of URLs (for in-process class loader usage).
*
* @return List of class path URLs.
@@ -190,6 +168,7 @@ public class HadoopClasspathUtils {
Collection<SearchDirectory> res = new ArrayList<>();
+ // 1. Add libraries from Hadoop distribution:
res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
@@ -204,8 +183,19 @@ public class HadoopClasspathUtils {
res.add(new SearchDirectory(new File(loc.mapred()),
new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
+ // 2. Add user provided libs:
res.addAll(parseUserLibs());
+ // 3. Add hadoop-impl module.
+ String home = systemOrEnv(IGNITE_HOME, null);
+
+ if (home != null) {
+ File dir = new File(home, "libs/ignite-hadoop-impl");
+
+ if (dir.exists())
+ res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, true));
+ }
+
return res;
}
@@ -353,13 +343,6 @@ public class HadoopClasspathUtils {
}
/**
- * @return Absolute path.
- */
- public String absolutePath() {
- return dir.getAbsolutePath();
- }
-
- /**
* @return Child files.
*/
public File[] files() throws IOException {
@@ -378,13 +361,6 @@ public class HadoopClasspathUtils {
else
return files;
}
-
- /**
- * @return {@code True} if wildcard can be used.
- */
- public boolean useWildcard() {
- return filter instanceof AcceptAllDirectoryFilter;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
index 0be8bf9..0ca2851 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
@@ -247,7 +247,8 @@ public class HadoopCommandLineTest extends GridCommonAbstractTest {
String ggClsPath = HadoopJob.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
HadoopJobTracker.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
- ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+ ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath()
+ + sep + U.getIgniteHome() + "/modules/hadoop-impl/target/classes";
ProcessBuilder res = new ProcessBuilder();