You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/07/01 12:10:44 UTC
[01/27] ignite git commit: 3184 : new java impl
Repository: ignite
Updated Branches:
refs/heads/ignite-3185 [created] 084abddb2
3184 : new java impl
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0c7b08c8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0c7b08c8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0c7b08c8
Branch: refs/heads/ignite-3185
Commit: 0c7b08c8f48a1d9f4f2610ae15c925f591900856
Parents: 8a14e47
Author: iveselovskiy <iv...@gridgain.com>
Authored: Fri Jun 17 22:34:08 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Fri Jun 17 22:34:08 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.sh | 13 +-
.../ignite/internal/util/IgniteUtils.java | 3 +-
.../processors/hadoop/HadoopClassLoader.java | 88 ++-----
.../processors/hadoop/HadoopClasspathMain.java | 244 +++++++++++++++++++
.../processors/hadoop/HadoopProcessor.java | 2 +-
5 files changed, 275 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 74b6608..2b17253 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -72,9 +72,16 @@ if [ "${USER_LIBS}" != "" ]; then
fi
if [ "${HADOOP_EDITION}" == "1" ]; then
- . "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
+ #. "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
+ IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain )
- if [ "${IGNITE_HADOOP_CLASSPATH}" != "" ]; then
- IGNITE_LIBS=${IGNITE_LIBS}${SEP}$IGNITE_HADOOP_CLASSPATH
+ # TODO: DEBUG:
+ echo "IGNITE_HADOOP_CLASSPATH=[${IGNITE_HADOOP_CLASSPATH}]"
+
+ if [ -z "${IGNITE_HADOOP_CLASSPATH}" ]; then
+ # Error already reported.
+ exit 1
+ else
+ IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
fi
fi
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index faf026b..c28e7bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4049,7 +4049,8 @@ public abstract class IgniteUtils {
public static void quietAndWarn(IgniteLogger log, Object msg, Object shortMsg) {
warn(log, msg);
- if (log.isQuiet())
+ if (log != null
+ && log.isQuiet())
quiet(false, shortMsg);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 4448b2d..8831137 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -17,11 +17,8 @@
package org.apache.ignite.internal.processors.hadoop;
-import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
-import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
@@ -57,6 +54,7 @@ import org.objectweb.asm.Opcodes;
import org.objectweb.asm.Type;
import org.objectweb.asm.commons.Remapper;
import org.objectweb.asm.commons.RemappingClassAdapter;
+import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.*;
/**
* Class loader allowing explicitly load classes without delegation to parent class loader.
@@ -171,9 +169,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
ldr = ldr.getParent();
}
}
- catch (Exception e) {
+ catch (Throwable t) {
U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
- "(native Hadoop methods might not work properly): " + e);
+ "(native Hadoop methods might not work properly): " + t);
}
}
@@ -439,39 +437,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return hasDot;
}
- /**
- * @param name Variable name.
- * @param dflt Default.
- * @return Value.
- */
- private static String getEnv(String name, String dflt) {
- String res = System.getProperty(name);
-
- if (F.isEmpty(res))
- res = System.getenv(name);
-
- return F.isEmpty(res) ? dflt : res;
- }
-
- /**
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws Exception {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory: " + dir);
-
- for (File file : files)
- res.add(file.toURI().toURL());
- }
+// /**
+// * @param name Variable name.
+// * @param dflt Default.
+// * @return Value.
+// */
+// private static String getEnv(String name, String dflt) {
+// String res = System.getProperty(name);
+//
+// if (F.isEmpty(res))
+// res = System.getenv(name);
+//
+// return F.isEmpty(res) ? dflt : res;
+// }
/**
* @param urls URLs.
@@ -499,13 +477,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * @return HADOOP_HOME Variable.
- */
- @Nullable public static String hadoopHome() {
- return getEnv("HADOOP_PREFIX", getEnv("HADOOP_HOME", null));
- }
-
- /**
* @return Collection of jar URLs.
* @throws IgniteCheckedException If failed.
*/
@@ -521,33 +492,10 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
if (hadoopUrls != null)
return hadoopUrls;
- hadoopUrls = new ArrayList<>();
-
- String hadoopPrefix = hadoopHome();
-
- if (F.isEmpty(hadoopPrefix))
- throw new IgniteCheckedException("Failed resolve Hadoop installation location. Either HADOOP_PREFIX or " +
- "HADOOP_HOME environment variables must be set.");
-
- String commonHome = getEnv("HADOOP_COMMON_HOME", hadoopPrefix + "/share/hadoop/common");
- String hdfsHome = getEnv("HADOOP_HDFS_HOME", hadoopPrefix + "/share/hadoop/hdfs");
- String mapredHome = getEnv("HADOOP_MAPRED_HOME", hadoopPrefix + "/share/hadoop/mapreduce");
-
try {
- addUrls(hadoopUrls, new File(commonHome + "/lib"), null);
- addUrls(hadoopUrls, new File(hdfsHome + "/lib"), null);
- addUrls(hadoopUrls, new File(mapredHome + "/lib"), null);
-
- addUrls(hadoopUrls, new File(hdfsHome), "hadoop-hdfs-");
-
- addUrls(hadoopUrls, new File(commonHome), "hadoop-common-");
- addUrls(hadoopUrls, new File(commonHome), "hadoop-auth-");
- addUrls(hadoopUrls, new File(commonHome + "/lib"), "hadoop-auth-");
-
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-common");
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-core");
+ hadoopUrls = getAsUrlList();
}
- catch (Exception e) {
+ catch (IOException e) {
throw new IgniteCheckedException(e);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
new file mode 100644
index 0000000..a83946f
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -0,0 +1,244 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+import static java.lang.System.out;
+import static java.lang.System.err;
+
+/**
+ * Main class to compose Hadoop classpath depending on the environment.
+ * Note that this class should not depend on any classes or libraries except the JDK default runtime.
+ */
+public class HadoopClasspathMain {
+ /**
+ *
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ final char sep = File.pathSeparatorChar;
+
+ List<String> cp = getAsProcessClasspath();
+
+ for (String s: cp) {
+ if (s != null && s.length() > 0) {
+ out.print(s);
+ out.print(sep);
+ }
+ }
+
+ out.println();
+ }
+
+ /**
+ *
+ * @return
+ */
+ private static List<String> getAsProcessClasspath() throws IOException {
+ Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+ List<String> list = new ArrayList<>(32);
+
+ for (DirAndMask dam: dams)
+ addAsJavaProcessClasspathElement(list, dam.file, dam.mask);
+
+ // Sort the classpath elements to make it more reliable.
+ Collections.sort(list);
+
+ return list;
+ }
+
+ /**
+ *
+ * @return
+ * @throws IOException
+ */
+ public static List<URL> getAsUrlList() throws IOException {
+ Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+ List<URL> list = new ArrayList<>(32);
+
+ for (DirAndMask dam: dams)
+ // Note that this procedure does not use '*' classpath patterns,
+ // but adds all the children explicitly:
+ addUrls(list, dam.file, dam.mask);
+
+ Collections.sort(list, new Comparator<URL>() {
+ @Override public int compare(URL u1, URL u2) {
+ String s1 = String.valueOf(u1);
+ String s2 = String.valueOf(u2);
+
+ return s1.compareTo(s2);
+ }
+ });
+
+ for (URL u: list)
+ err.println(u);
+
+ return list;
+ }
+
+ /**
+ * @param res Result.
+ * @param dir Directory.
+ * @param startsWith Starts with prefix.
+ * @throws MalformedURLException If failed.
+ */
+ private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return startsWith == null || name.startsWith(startsWith);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory: " + dir);
+
+ for (File file : files)
+ res.add(file.toURI().toURL());
+ }
+
+
+ /**
+ * @param res Result.
+ * @param dir Directory.
+ * @param startsWith Starts with prefix.
+ * @throws MalformedURLException If failed.
+ */
+ private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith) throws IOException {
+ if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
+ throw new IOException("Path is not an existing readable directory: " + dir);
+
+ if (startsWith == null)
+ res.add(dir.getAbsolutePath() + File.separator + '*');
+ else {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith(startsWith);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory: " + dir);
+
+ for (File file : files)
+ res.add(file.getAbsolutePath());
+ }
+ }
+
+ /**
+ * @return HADOOP_HOME Variable.
+ */
+ public static String hadoopHome() {
+ String prefix = getEnv("HADOOP_PREFIX", null);
+
+ return getEnv("HADOOP_HOME", prefix);
+ }
+
+ /**
+ *
+ * @return
+ * @throws FileNotFoundException
+ */
+ public static Collection<DirAndMask> getClasspathBaseDirectories() throws FileNotFoundException {
+ final String hadoopHome = hadoopHome();
+
+ String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
+ String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
+ String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+
+ Collection<DirAndMask> c = new ArrayList<>();
+
+ c.add(new DirAndMask(new File(commonHome, "lib"), null));
+ c.add(new DirAndMask(new File(hdfsHome, "lib"), null));
+ c.add(new DirAndMask(new File(mapredHome, "lib"), null));
+
+ c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
+ c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
+ c.add(new DirAndMask(new File(commonHome, "lib"), "hadoop-auth-"));
+
+ c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
+
+ c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-common"));
+ c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-core"));
+
+ return c;
+ }
+
+ public static class DirAndMask {
+ DirAndMask(File f, String m) {
+ file = f;
+ mask = m;
+ }
+ public final File file;
+ public final String mask;
+ }
+
+ /**
+ * Checks if the variable is empty.
+ *
+ * @param envVarName Environment variable name.
+ * @param hadoopHome The current value.
+ * @param expHadoopHomeRelativePath The path relative to Hadoop home.
+ * @throws FileNotFoundException If the value is empty.
+ */
+ private static String resolveLocation(String envVarName, String hadoopHome,
+ String expHadoopHomeRelativePath) throws FileNotFoundException {
+ String val = getEnv(envVarName, null);
+
+ if (val == null) {
+ // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
+ if (!isExistingDirectory(hadoopHome))
+ throw new FileNotFoundException("Failed to resolve Hadoop installation location. " +
+ envVarName + " or HADOOP_HOME environment variable should be set.");
+
+ val = hadoopHome + expHadoopHomeRelativePath;
+ }
+
+ if (!isExistingDirectory(val))
+ throw new FileNotFoundException("Failed to resolve Hadoop location. [path=" + val + ']');
+
+ err.println(envVarName + " resolved to " + val);
+
+ return val;
+ }
+
+ /**
+ * Note that this method does not treat empty value as an absent value.
+ *
+ * @param name Variable name.
+ * @param dflt Default.
+ * @return Value.
+ */
+ private static String getEnv(String name, String dflt) {
+ String res = System.getProperty(name);
+
+ if (res == null)
+ res = System.getenv(name);
+
+ return res == null ? dflt : res;
+ }
+
+ /**
+ * Answers if the given path denotes existing directory.
+ *
+ * @param path The directory path.
+ * @return 'true' if the given path denotes an existing directory.
+ */
+ private static boolean isExistingDirectory(String path) {
+ if (path == null)
+ return false;
+
+ Path p = Paths.get(path);
+
+ return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index 7292c78..a502c3e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -33,8 +33,8 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopHome;
import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
+import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.hadoopHome;
/**
* Hadoop processor.
[05/27] ignite git commit: Merge branch 'gridgain-7.5.27' of
https://github.com/gridgain/apache-ignite into ignite-3184
Posted by vo...@apache.org.
Merge branch 'gridgain-7.5.27' of https://github.com/gridgain/apache-ignite into ignite-3184
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d27924c9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d27924c9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d27924c9
Branch: refs/heads/ignite-3185
Commit: d27924c9352a774d820c3bf81fe9186d5c39b598
Parents: aa5f5a6 dc28c82
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Jun 21 15:41:49 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Jun 21 15:41:49 2016 +0300
----------------------------------------------------------------------
modules/clients/pom.xml | 8 -
.../client/ClientDefaultCacheSelfTest.java | 119 +-
.../ignite/internal/client/ClientHttpTask.java | 33 +-
.../ignite/internal/client/ClientTcpTask.java | 10 +-
.../integration/ClientAbstractSelfTest.java | 92 +-
.../JettyRestProcessorAbstractSelfTest.java | 1428 ++++++++++++------
.../internal/processors/rest/SimplePerson.java | 74 +
modules/core/pom.xml | 15 +-
.../apache/ignite/internal/LessNamingBean.java | 28 +
.../cache/query/GridCacheSqlIndexMetadata.java | 3 +-
.../cache/query/GridCacheSqlMetadata.java | 3 +-
.../processors/igfs/IgfsDeleteWorker.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 99 +-
.../igfs/client/IgfsClientAbstractCallable.java | 17 -
.../igfs/client/IgfsClientDeleteCallable.java | 7 -
.../igfs/client/IgfsClientMkdirsCallable.java | 6 -
.../igfs/client/IgfsClientRenameCallable.java | 6 -
.../igfs/client/IgfsClientSetTimesCallable.java | 7 -
.../igfs/client/IgfsClientUpdateCallable.java | 6 -
.../meta/IgfsClientMetaFileUnlockCallable.java | 130 --
.../internal/util/IgniteExceptionRegistry.java | 5 +-
.../ignite/internal/visor/cache/VisorCache.java | 10 +-
.../cache/VisorCacheAffinityConfiguration.java | 5 +-
.../cache/VisorCacheAggregatedMetrics.java | 3 +-
.../visor/cache/VisorCacheConfiguration.java | 3 +-
.../cache/VisorCacheDefaultConfiguration.java | 5 +-
.../cache/VisorCacheEvictionConfiguration.java | 5 +-
.../internal/visor/cache/VisorCacheMetrics.java | 3 +-
.../cache/VisorCacheNearConfiguration.java | 5 +-
.../visor/cache/VisorCachePartition.java | 3 +-
.../visor/cache/VisorCachePartitions.java | 3 +-
.../cache/VisorCacheQueryConfiguration.java | 3 +-
.../visor/cache/VisorCacheQueryMetrics.java | 5 +-
.../cache/VisorCacheRebalanceConfiguration.java | 5 +-
.../cache/VisorCacheStoreConfiguration.java | 3 +-
.../cache/VisorCacheTypeFieldMetadata.java | 3 +-
.../visor/cache/VisorCacheTypeMetadata.java | 3 +-
.../internal/visor/cache/VisorCacheV4.java | 124 ++
.../visor/compute/VisorGatewayTask.java | 361 +++++
.../internal/visor/debug/VisorThreadInfo.java | 5 +-
.../visor/debug/VisorThreadLockInfo.java | 5 +-
.../internal/visor/event/VisorGridEvent.java | 5 +-
.../internal/visor/file/VisorFileBlock.java | 5 +-
.../ignite/internal/visor/igfs/VisorIgfs.java | 5 +-
.../internal/visor/igfs/VisorIgfsEndpoint.java | 5 +-
.../internal/visor/igfs/VisorIgfsMetrics.java | 5 +-
.../visor/igfs/VisorIgfsProfilerEntry.java | 5 +-
.../VisorIgfsProfilerUniformityCounters.java | 5 +-
.../visor/log/VisorLogSearchResult.java | 5 +-
.../visor/node/VisorAtomicConfiguration.java | 5 +-
.../visor/node/VisorBasicConfiguration.java | 5 +-
.../node/VisorExecutorServiceConfiguration.java | 5 +-
.../visor/node/VisorGridConfiguration.java | 5 +-
.../visor/node/VisorIgfsConfiguration.java | 3 +-
.../visor/node/VisorLifecycleConfiguration.java | 5 +-
.../visor/node/VisorMetricsConfiguration.java | 5 +-
.../visor/node/VisorNodeDataCollectorJob.java | 28 +-
.../node/VisorNodeDataCollectorTaskResult.java | 5 +-
.../node/VisorPeerToPeerConfiguration.java | 5 +-
.../visor/node/VisorRestConfiguration.java | 5 +-
.../node/VisorSegmentationConfiguration.java | 5 +-
.../visor/node/VisorSpisConfiguration.java | 5 +-
.../node/VisorTransactionConfiguration.java | 5 +-
.../internal/visor/query/VisorQueryField.java | 5 +-
.../internal/visor/query/VisorQueryResult.java | 5 +-
.../internal/visor/util/VisorTaskUtils.java | 10 +-
.../plugin/security/SecurityPermissionSet.java | 5 +-
.../ignite/plugin/security/SecuritySubject.java | 5 +-
modules/rest-http/pom.xml | 25 +-
.../http/jetty/GridJettyJsonConfig.java | 195 ---
.../http/jetty/GridJettyObjectMapper.java | 274 ++++
.../http/jetty/GridJettyRestHandler.java | 127 +-
parent/pom.xml | 1 +
73 files changed, 2271 insertions(+), 1172 deletions(-)
----------------------------------------------------------------------
[14/27] ignite git commit: IGNITE-3185
Posted by vo...@apache.org.
IGNITE-3185
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c4017097
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c4017097
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c4017097
Branch: refs/heads/ignite-3185
Commit: c4017097bd443069090626acba4772de01051f98
Parents: 9cc1607
Author: iveselovskiy <iv...@gridgain.com>
Authored: Fri Jun 24 17:26:32 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Fri Jun 24 17:26:32 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 148 ++++++++++++++-----
1 file changed, 115 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c4017097/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index bb7ccd3..acf19bd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -163,21 +163,130 @@ public class HadoopClasspathUtils {
this.hdfs = hdfs;
this.mapred = mapred;
}
+
+ /**
+ * Answers if all the base directories are defined.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+ */
+ public boolean isDefined() {
+ return common != null && hdfs != null && mapred != null;
+ }
+
+ /**
+ * Answers if all the base directories exist.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+ */
+ public boolean exists() {
+ return isExistingDirectory(common)
+ && isExistingDirectory(hdfs)
+ && isExistingDirectory(mapred);
+ }
+
+ /**
+ * Checks if all the base directories exist.
+ *
+ * @return this reference.
+ * @throws IOException if any of the base directories does not exist.
+ */
+ public HadoopLocations existsOrException() throws IOException {
+ if (!isExistingDirectory(common))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_COMMON_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(hdfs))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_HDFS_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(mapred))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_MAPRED_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ return this;
+ }
+ }
+
+ /**
+ * Gets locations from the environment.
+ *
+ * @return The locations as determined from the environment.
+ */
+ private static HadoopLocations getEnvHadoopLocations() {
+ return new HadoopLocations(
+ hadoopHome(),
+ getEnv("HADOOP_COMMON_HOME", null),
+ getEnv("HADOOP_HDFS_HOME", null),
+ getEnv("HADOOP_MAPRED_HOME", null)
+ );
+ }
+
+ /**
+ * Gets locations assuming Apache Hadoop distribution layout.
+ *
+ * @return The locations as for Apache distribution.
+ */
+ private static HadoopLocations getApacheHadoopLocations(String hadoopHome) {
+ return new HadoopLocations(hadoopHome,
+ hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs",
+ hadoopHome + "/share/hadoop/mapreduce");
+ }
+
+ /** HDP Hadoop locations. */
+ private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-hdfs-client/",
+ "/usr/hdp/current/hadoop-mapreduce-client/");
+
+ /**
+ * HDP locations relative to an arbitrary Hadoop home.
+ *
+ * @param hadoopHome The hadoop home.
+ * @return The locations.
+ */
+ private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
+ return new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client/",
+ hadoopHome + "/../hadoop-mapreduce-client/");
}
/**
- * Gets Hadoop locations.
+ * Gets the existing Hadoop locations, if any.
*
- * @return The Hadoop locations, never null.
+ * @return Existing Hadoop locations.
+ * @throws IOException If no existing location found.
*/
public static HadoopLocations getHadoopLocations() throws IOException {
+ // 1. Try locations defined in System properties or environment:
+ HadoopLocations loc = getEnvHadoopLocations();
+
+ if (loc.isDefined())
+ return loc.existsOrException();
+
final String hadoopHome = hadoopHome();
- String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
- String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
- String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+ if (hadoopHome != null) {
+ // If home is defined, it must exist:
+ if (!isExistingDirectory(hadoopHome))
+ throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
+ + hadoopHome + ']');
+
+ // 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
+ loc = getApacheHadoopLocations(hadoopHome);
- return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+ if (loc.exists())
+ return loc;
+
+ // 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
+ loc = getHdpLocationsRelative(hadoopHome);
+
+ return loc.existsOrException();
+ }
+
+ // 4. Try absolute HDP (Hortonworks) location:
+ return HDP_HADOOP_LOCATIONS.existsOrException();
}
/**
@@ -229,33 +338,6 @@ public class HadoopClasspathUtils {
}
/**
- * Resolves a Hadoop location directory.
- *
- * @param envVarName Environment variable name. The value denotes the location path.
- * @param hadoopHome Hadoop home location, may be null.
- * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
- * @throws IOException If the value cannot be resolved to an existing directory.
- */
- private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
- throws IOException {
- String val = getEnv(envVarName, null);
-
- if (val == null) {
- // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!isExistingDirectory(hadoopHome))
- throw new IOException("Failed to resolve Hadoop installation location. " +
- envVarName + " or HADOOP_HOME environment variable should be set.");
-
- val = hadoopHome + expHadoopHomeRelativePath;
- }
-
- if (!isExistingDirectory(val))
- throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
-
- return val;
- }
-
- /**
* Note that this method does not treat empty value as an absent value.
*
* @param name Variable name.
[23/27] ignite git commit: IGNITE-3185: Review comments.
Posted by vo...@apache.org.
IGNITE-3185: Review comments.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/057654af
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/057654af
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/057654af
Branch: refs/heads/ignite-3185
Commit: 057654af627defdec8b278d4e134ca663685bd3b
Parents: 04ea922
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Jun 29 18:28:30 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Jun 29 18:28:30 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 52 +++++++++++---------
1 file changed, 29 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/057654af/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index f84b093..121fcab 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -127,36 +127,42 @@ public class HadoopClasspathUtils {
throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
"variable points to nonexistent directory: " + hadoopHome);
- // Create the list of probes and then iterate over them.
- List<HadoopLocations> probes = new ArrayList<>();
+ // Probe Apache Hadoop.
+ HadoopLocations res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs",
+ hadoopHome + "/share/hadoop/mapreduce"
+ );
- // Probe for Apache Hadoop.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/share/hadoop/common",
- hadoopHome + "/share/hadoop/hdfs", hadoopHome + "/share/hadoop/mapreduce"));
-
- // Probe for CDH with normal HADOOP_HOME directory.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs", hadoopHome + "/../hadoop-mapreduce"));
+ if (res.valid())
+ return res;
- // Probe for CDH with HADOOP_HOME set to parent directory.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop",
- hadoopHome + "/hadoop-hdfs", hadoopHome + "/hadoop-mapreduce"));
+ // Probe CDH.
+ res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome,
+ hadoopHome + "/../hadoop-hdfs",
+ hadoopHome + "/../hadoop-mapreduce"
+ );
- // Probe for HDP with normal HADOOP_HOME directory.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs-client", hadoopHome + "/../hadoop-mapreduce-client"));
+ if (res.valid())
+ return res;
- // Probe for HDP with HADOOP_HOME set to parent directory.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop-client,",
- hadoopHome + "/hadoop-hdfs-client", hadoopHome + "/hadoop-mapreduce-client"));
+ // Probe HDP.
+ res = new HadoopLocations(
+ hadoopHome,
+ hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client",
+ hadoopHome + "/../hadoop-mapreduce-client"
+ );
- for (HadoopLocations probe : probes) {
- if (probe.valid())
- return probe;
- }
+ if (res.valid())
+ return res;
+ // Failed.
throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
- "points to invalid directory: " + hadoopHome);
+ "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome);
}
else {
// Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
[11/27] ignite git commit: IGNITE-3184: Review.
Posted by vo...@apache.org.
IGNITE-3184: Review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/61b12c60
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/61b12c60
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/61b12c60
Branch: refs/heads/ignite-3185
Commit: 61b12c6048ff3566c76f4330c66ddb8c297ed820
Parents: 7bbaafc
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 23 14:56:30 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 23 14:56:30 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/util/IgniteUtils.java | 2 +-
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopClasspathMain.java | 30 +++++++---
.../processors/hadoop/HadoopClasspathUtils.java | 59 +++++++++++++-------
4 files changed, 63 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b12c60/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 8431645..fa18923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4049,7 +4049,7 @@ public abstract class IgniteUtils {
public static void quietAndWarn(IgniteLogger log, Object msg, Object shortMsg) {
warn(log, msg);
- if (log != null && log.isQuiet())
+ if (log.isQuiet())
quiet(false, shortMsg);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b12c60/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index e614586..c0251eb 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
hadoopUrls = HadoopClasspathUtils.getAsUrlList();
}
catch (IOException e) {
- throw new IgniteCheckedException(e);
+ throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e);
}
hadoopJars = hadoopUrls;
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b12c60/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
index 0a0b422..fb0de7c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -1,10 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.hadoop;
import java.util.List;
-import static java.lang.System.out;
-import static java.lang.System.err;
-
/**
* Main class to compose Hadoop classpath depending on the environment.
* This class is designed to be independent on any Ignite classes as possible.
@@ -18,7 +32,8 @@ public class HadoopClasspathMain {
*/
public static void main(String[] args) throws Exception {
if (args.length < 1) {
- err.println("Path separator must be passed as the 1st argument.");
+ // TODO: TO exception.
+ System.err.println("Path separator must be passed as the 1st argument.");
System.exit(1);
}
@@ -28,12 +43,13 @@ public class HadoopClasspathMain {
List<String> cp = HadoopClasspathUtils.getAsProcessClasspath();
for (String s: cp) {
+ // TODO: Move null-checks to getAsProcessClasspath().
if (s != null && s.length() > 0) {
- out.print(s);
- out.print(sep);
+ System.out.print(s);
+ System.out.print(sep);
}
}
- out.println();
+ System.out.println();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b12c60/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index e122913..7ef2f60 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -1,7 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.hadoop;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -13,8 +29,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import static java.lang.System.err;
-
/**
* Hadoop classpath utilities.
*/
@@ -23,7 +37,7 @@ public class HadoopClasspathUtils {
* Gets Hadoop class path as list of classpath elements for process.
*
* @return List of the class path elements.
- * @throws IOException On error.
+ * @throws IOException If failed.
*/
static List<String> getAsProcessClasspath() throws IOException {
Collection<DirAndMask> dams = getClasspathBaseDirectories();
@@ -40,7 +54,7 @@ public class HadoopClasspathUtils {
* Gets Hadoop class path as a list of URLs (for in-process class loader usage).
*
* @return List of class path URLs.
- * @throws IOException On error.
+ * @throws IOException If failed.
*/
public static List<URL> getAsUrlList() throws IOException {
Collection<DirAndMask> dams = getClasspathBaseDirectories();
@@ -61,7 +75,7 @@ public class HadoopClasspathUtils {
* @param res Result.
* @param dir Directory.
* @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
+ * @throws IOException If failed.
*/
private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
File[] files = dir.listFiles(new FilenameFilter() {
@@ -73,20 +87,26 @@ public class HadoopClasspathUtils {
if (files == null)
throw new IOException("Path is not a directory. [dir=" + dir + ']');
- for (File file : files)
- res.add(file.toURI().toURL());
+ for (File file : files) {
+ try {
+ res.add(file.toURI().toURL());
+ }
+ catch (MalformedURLException e) {
+ throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ }
+ }
}
-
/**
* Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
*
* @param res Result.
* @param dir Directory.
* @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
+ * @throws IOException If failed.
*/
- private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith) throws IOException {
+ private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith)
+ throws IOException {
if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
throw new IOException("Path is not an existing readable directory. [dir=" + dir + ']');
@@ -120,9 +140,9 @@ public class HadoopClasspathUtils {
* Gets base directories to discover classpath elements in.
*
* @return Collection of directory and mask pairs.
- * @throws FileNotFoundException if a mandatory classpath location is not found.
+ * @throws IOException if a mandatory classpath location is not found.
*/
- private static Collection<DirAndMask> getClasspathBaseDirectories() throws FileNotFoundException {
+ private static Collection<DirAndMask> getClasspathBaseDirectories() throws IOException {
final String hadoopHome = hadoopHome();
String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
@@ -174,26 +194,23 @@ public class HadoopClasspathUtils {
* @param envVarName Environment variable name. The value denotes the location path.
* @param hadoopHome Hadoop home location, may be null.
* @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
- * @throws FileNotFoundException If the value cannot be resolved to an existing directory.
+ * @throws IOException If the value cannot be resolved to an existing directory.
*/
- private static String resolveLocation(String envVarName, String hadoopHome,
- String expHadoopHomeRelativePath) throws FileNotFoundException {
+ private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
+ throws IOException {
String val = getEnv(envVarName, null);
if (val == null) {
// The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
if (!isExistingDirectory(hadoopHome))
- throw new FileNotFoundException("Failed to resolve Hadoop installation location. " +
+ throw new IOException("Failed to resolve Hadoop installation location. " +
envVarName + " or HADOOP_HOME environment variable should be set.");
val = hadoopHome + expHadoopHomeRelativePath;
}
if (!isExistingDirectory(val))
- throw new FileNotFoundException("Failed to resolve Hadoop location. [path=" + val + ']');
-
- // Print diagnostic output:
- err.println(envVarName + " resolved to " + val);
+ throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
return val;
}
[12/27] ignite git commit: IGNITE-3184: corrections by review results.
Posted by vo...@apache.org.
IGNITE-3184: corrections by review results.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9cc1607f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9cc1607f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9cc1607f
Branch: refs/heads/ignite-3185
Commit: 9cc1607fb87123d6d0f1c1ab469b51abf75394d6
Parents: 61b12c6
Author: iveselovskiy <iv...@gridgain.com>
Authored: Thu Jun 23 17:05:30 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Thu Jun 23 17:05:30 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathMain.java | 15 ++---
.../processors/hadoop/HadoopClasspathUtils.java | 66 ++++++++++++++++----
.../processors/hadoop/HadoopProcessor.java | 45 ++++++-------
3 files changed, 80 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9cc1607f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
index fb0de7c..b1422d4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -31,23 +31,16 @@ public class HadoopClasspathMain {
* @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
*/
public static void main(String[] args) throws Exception {
- if (args.length < 1) {
- // TODO: TO exception.
- System.err.println("Path separator must be passed as the 1st argument.");
-
- System.exit(1);
- }
+ if (args.length < 1)
+ throw new IllegalArgumentException("Path separator must be passed as the 1st argument.");
final String sep = args[0];
List<String> cp = HadoopClasspathUtils.getAsProcessClasspath();
for (String s: cp) {
- // TODO: Move null-checks to getAsProcessClasspath().
- if (s != null && s.length() > 0) {
- System.out.print(s);
- System.out.print(sep);
- }
+ System.out.print(s);
+ System.out.print(sep);
}
System.out.println();
http://git-wip-us.apache.org/repos/asf/ignite/blob/9cc1607f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 7ef2f60..bb7ccd3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -130,38 +130,78 @@ public class HadoopClasspathUtils {
/**
* @return HADOOP_HOME Variable.
*/
- public static String hadoopHome() {
+ private static String hadoopHome() {
String prefix = getEnv("HADOOP_PREFIX", null);
return getEnv("HADOOP_HOME", prefix);
}
/**
- * Gets base directories to discover classpath elements in.
+ * Simple structure to hold Hadoop directory locations.
+ */
+ public static class HadoopLocations {
+ /** HADOOP_HOME, may be null. */
+ public final String home;
+ /** HADOOP_COMMON_HOME */
+ public final String common;
+ /** HADOOP_HDFS_HOME */
+ public final String hdfs;
+ /** HADOOP_MAPRED_HOME */
+ public final String mapred;
+
+ /**
+ * Constructor.
+ *
+ * @param home HADOOP_HOME
+ * @param common HADOOP_COMMON_HOME
+ * @param hdfs HADOOP_HDFS_HOME
+ * @param mapred HADOOP_MAPRED_HOME
+ */
+ HadoopLocations(String home, String common, String hdfs, String mapred) {
+ this.home = home;
+ this.common = common;
+ this.hdfs = hdfs;
+ this.mapred = mapred;
+ }
+ }
+
+ /**
+ * Gets Hadoop locations.
*
- * @return Collection of directory and mask pairs.
- * @throws IOException if a mandatory classpath location is not found.
+ * @return The Hadoop locations, never null.
*/
- private static Collection<DirAndMask> getClasspathBaseDirectories() throws IOException {
+ public static HadoopLocations getHadoopLocations() throws IOException {
final String hadoopHome = hadoopHome();
String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+ return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+ }
+
+ /**
+ * Gets base directories to discover classpath elements in.
+ *
+ * @return Collection of directory and mask pairs.
+ * @throws IOException if a mandatory classpath location is not found.
+ */
+ private static Collection<DirAndMask> getClasspathBaseDirectories() throws IOException {
+ HadoopLocations loc = getHadoopLocations();
+
Collection<DirAndMask> c = new ArrayList<>();
- c.add(new DirAndMask(new File(commonHome, "lib"), null));
- c.add(new DirAndMask(new File(hdfsHome, "lib"), null));
- c.add(new DirAndMask(new File(mapredHome, "lib"), null));
+ c.add(new DirAndMask(new File(loc.common, "lib"), null));
+ c.add(new DirAndMask(new File(loc.hdfs, "lib"), null));
+ c.add(new DirAndMask(new File(loc.mapred, "lib"), null));
- c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
- c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
+ c.add(new DirAndMask(new File(loc.common), "hadoop-common-"));
+ c.add(new DirAndMask(new File(loc.common), "hadoop-auth-"));
- c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
+ c.add(new DirAndMask(new File(loc.hdfs), "hadoop-hdfs-"));
- c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-common"));
- c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-core"));
+ c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-common"));
+ c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-core"));
return c;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9cc1607f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index 98e1fc5..c4cd143 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.hadoop;
+import java.io.IOException;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.atomic.AtomicInteger;
@@ -32,6 +33,7 @@ import org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopEmbeddedT
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.processors.hadoop.HadoopClasspathUtils.HadoopLocations;
/**
* Hadoop processor.
@@ -71,36 +73,35 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
- if (HadoopClasspathUtils.hadoopHome() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + HadoopClasspathUtils.hadoopHome());
+ try {
+ HadoopLocations loc = HadoopClasspathUtils.getHadoopLocations();
- boolean ok = false;
+ if (loc.home != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home);
- try { // Check for Hadoop installation.
- HadoopClassLoader.hadoopUrls();
-
- ok = true;
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.common);
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfs);
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapred);
}
- catch (IgniteCheckedException e) {
- U.quietAndWarn(log, e.getMessage());
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe);
}
- if (ok) {
- hctx = new HadoopContext(
- ctx,
- cfg,
- new HadoopJobTracker(),
- new HadoopEmbeddedTaskExecutor(),
- // TODO: IGNITE-404: Uncomment when fixed.
- //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
- new HadoopShuffle());
+ HadoopClassLoader.hadoopUrls();
+ hctx = new HadoopContext(
+ ctx,
+ cfg,
+ new HadoopJobTracker(),
+ new HadoopEmbeddedTaskExecutor(),
+ // TODO: IGNITE-404: Uncomment when fixed.
+ //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
+ new HadoopShuffle());
- for (HadoopComponent c : hctx.components())
- c.start(hctx);
+ for (HadoopComponent c : hctx.components())
+ c.start(hctx);
- hadoop = new HadoopImpl(this);
- }
+ hadoop = new HadoopImpl(this);
}
/** {@inheritDoc} */
[03/27] ignite git commit: IGNITE-3184: rewritten with pure-java
classpath composition.
Posted by vo...@apache.org.
IGNITE-3184: rewritten with pure-java classpath composition.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f0d08f86
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f0d08f86
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f0d08f86
Branch: refs/heads/ignite-3185
Commit: f0d08f86fd277aa549ba49d28049e7880e0dd18e
Parents: 0c7b08c
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Jun 20 19:27:24 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Jun 20 19:27:24 2016 +0300
----------------------------------------------------------------------
bin/include/hadoop-classpath.bat | 73 ------
bin/include/hadoop-classpath.sh | 65 ------
bin/include/setenv.bat | 8 +-
bin/include/setenv.sh | 18 +-
.../processors/hadoop/HadoopClassLoader.java | 17 +-
.../processors/hadoop/HadoopClasspathMain.java | 231 ++----------------
.../processors/hadoop/HadoopClasspathUtils.java | 232 +++++++++++++++++++
.../processors/hadoop/HadoopProcessor.java | 9 +-
8 files changed, 261 insertions(+), 392 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/bin/include/hadoop-classpath.bat
----------------------------------------------------------------------
diff --git a/bin/include/hadoop-classpath.bat b/bin/include/hadoop-classpath.bat
deleted file mode 100644
index dd293a7..0000000
--- a/bin/include/hadoop-classpath.bat
+++ /dev/null
@@ -1,73 +0,0 @@
-::
-:: Licensed to the Apache Software Foundation (ASF) under one or more
-:: contributor license agreements. See the NOTICE file distributed with
-:: this work for additional information regarding copyright ownership.
-:: The ASF licenses this file to You under the Apache License, Version 2.0
-:: (the "License"); you may not use this file except in compliance with
-:: the License. You may obtain a copy of the License at
-::
-:: http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing, software
-:: distributed under the License is distributed on an "AS IS" BASIS,
-:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-:: See the License for the specific language governing permissions and
-:: limitations under the License.
-::
-
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-:: Hadoop class path resolver.
-:: Requires environment variables 'HADOOP_PREFIX' or 'HADOOP_HOME'
-:: to be set.
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-:: Turn off script echoing.
-@echo off
-
-:: Check if environment passes deprecated HADOOP_HOME.
-if not defined HADOOP_PREFIX set HADOOP_PREFIX=%HADOOP_HOME%
-
-:: Exit if we cannot find Hadoop installation directory.
-if not defined HADOOP_PREFIX goto :eof
-
-:: Trim quotes.
-set HADOOP_PREFIX=%HADOOP_PREFIX:"=%
-
-:: Trim slashes.
-if %HADOOP_PREFIX:~-1,1% == \ (
- set HADOOP_PREFIX=%HADOOP_PREFIX:~0,-1%
-)
-
-::
-:: Setting the rest of Hadoop environment variables.
-::
-
-if not defined HADOOP_COMMON_HOME set HADOOP_COMMON_HOME=%HADOOP_PREFIX%\share\hadoop\common
-if not defined HADOOP_HDFS_HOME set HADOOP_HDFS_HOME=%HADOOP_PREFIX%\share\hadoop\hdfs
-if not defined HADOOP_MAPRED_HOME set HADOOP_MAPRED_HOME=%HADOOP_PREFIX%\share\hadoop\mapreduce
-
-::
-:: Libraries included in classpath.
-::
-
-set CP=%HADOOP_COMMON_HOME%\lib\*;%HADOOP_MAPRED_HOME%\lib\*;%HADOOP_MAPRED_HOME%\lib\*
-
-:: hadoop-auth-* jar can be located either in home or in home/lib directory, depending on the hadoop version.
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\hadoop-auth-* ^>nul 2^>^&1') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\lib\hadoop-auth-* ^>nul 2^>^&1') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\hadoop-common-*') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_HDFS_HOME%\hadoop-hdfs-*') do call :concat %HADOOP_HDFS_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_MAPRED_HOME%\hadoop-mapreduce-client-common-*') do call :concat %HADOOP_MAPRED_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_MAPRED_HOME%\hadoop-mapreduce-client-core-*') do call :concat %HADOOP_MAPRED_HOME%\%%f
-
-:: Export result.
-set IGNITE_HADOOP_CLASSPATH=%CP%
-
-:: Exit.
-goto :eof
-
-:: Function that adds jar dependency into classpath.
-:concat
- set file=%1
- if %file:~-9,9% neq tests.jar set CP=%CP%;%1
-goto :eof
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/bin/include/hadoop-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/include/hadoop-classpath.sh b/bin/include/hadoop-classpath.sh
deleted file mode 100755
index 3a0aeb1..0000000
--- a/bin/include/hadoop-classpath.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-####################################################################
-# Hadoop class path resolver.
-# Requires environment variables 'HADOOP_PREFIX' or 'HADOOP_HOME'
-# to be set. If they are both undefined , tries to read them from
-# from '/etc/default/hadoop' file. The final results are printed
-# into standard output.
-####################################################################
-
-# Resolve constants.
-HADOOP_DEFAULTS="/etc/default/hadoop"
-HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_HOME}
-
-# Try get all variables from Hadoop default environment config
-# if they have not been passed into the script.
-if [[ -z "$HADOOP_PREFIX" && -f "$HADOOP_DEFAULTS" ]]; then
- source "$HADOOP_DEFAULTS"
-fi
-
-# Return if Hadoop couldn't be found.
-[ -z "$HADOOP_PREFIX" ] && return
-
-#
-# Resolve the rest of Hadoop environment variables.
-#
-
-HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME-"${HADOOP_PREFIX}/share/hadoop/common"}
-HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME-"${HADOOP_PREFIX}/share/hadoop/hdfs"}
-HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME-"${HADOOP_PREFIX}/share/hadoop/mapreduce"}
-
-#
-# Calculate classpath string with required Hadoop libraries.
-#
-
-# Add all Hadoop libs.
-IGNITE_HADOOP_CLASSPATH="${HADOOP_COMMON_HOME}/lib/*${SEP}${HADOOP_MAPRED_HOME}/lib/*${SEP}${HADOOP_MAPRED_HOME}/lib/*"
-
-# Skip globbing pattern if it cannot be resolved.
-shopt -s nullglob
-
-# Add jars to classpath excluding tests.
-# hadoop-auth-* jar can be located either in home or in home/lib directory, depending on the hadoop version.
-for file in ${HADOOP_HDFS_HOME}/hadoop-hdfs-* \
- ${HADOOP_COMMON_HOME}/hadoop-{common,auth}-* \
- ${HADOOP_COMMON_HOME}/lib/hadoop-auth-* \
- ${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-{common,core}-*; do
- [[ "$file" != *-tests.jar ]] && IGNITE_HADOOP_CLASSPATH=${IGNITE_HADOOP_CLASSPATH}${SEP}${file}
-done
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 811bc95..7022a36 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -23,7 +23,7 @@
:: in other scripts to set classpath using exported IGNITE_LIBS variable.
::
-@echo off
+:: @echo off
:: USER_LIBS variable can optionally contain user's JARs/libs.
:: set USER_LIBS=
@@ -47,11 +47,9 @@ if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
-if "%HADOOP_EDITION%" == "1" call "%SCRIPTS_HOME%\include\hadoop-classpath.bat"
+FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
-set COMMON_HOME_LIB=%HADOOP_COMMON_HOME%\lib
-
-if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
+if "[%IGNITE_HADOOP_CLASSPATH%]" == "[]" exit 1
set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 2b17253..1f51b59 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -72,16 +72,16 @@ if [ "${USER_LIBS}" != "" ]; then
fi
if [ "${HADOOP_EDITION}" == "1" ]; then
- #. "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
- IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain )
+ IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \
+ org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
- # TODO: DEBUG:
- echo "IGNITE_HADOOP_CLASSPATH=[${IGNITE_HADOOP_CLASSPATH}]"
+ statusCode=${?}
- if [ -z "${IGNITE_HADOOP_CLASSPATH}" ]; then
- # Error already reported.
- exit 1
- else
- IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
+ if [ "${statusCode}" -ne 0 ]; then
+ exit ${statusCode}
fi
+
+ unset statusCode
+
+ IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
fi
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 8831137..8b6ea34 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -54,7 +54,6 @@ import org.objectweb.asm.Opcodes;
import org.objectweb.asm.Type;
import org.objectweb.asm.commons.Remapper;
import org.objectweb.asm.commons.RemappingClassAdapter;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.*;
/**
* Class loader allowing explicitly load classes without delegation to parent class loader.
@@ -437,20 +436,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return hasDot;
}
-// /**
-// * @param name Variable name.
-// * @param dflt Default.
-// * @return Value.
-// */
-// private static String getEnv(String name, String dflt) {
-// String res = System.getProperty(name);
-//
-// if (F.isEmpty(res))
-// res = System.getenv(name);
-//
-// return F.isEmpty(res) ? dflt : res;
-// }
-
/**
* @param urls URLs.
* @return URLs.
@@ -493,7 +478,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return hadoopUrls;
try {
- hadoopUrls = getAsUrlList();
+ hadoopUrls = HadoopClasspathUtils.getAsUrlList();
}
catch (IOException e) {
throw new IgniteCheckedException(e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
index a83946f..0a0b422 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -1,32 +1,31 @@
package org.apache.ignite.internal.processors.hadoop;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.*;
+import java.util.List;
import static java.lang.System.out;
import static java.lang.System.err;
/**
* Main class to compose Hadoop classpath depending on the environment.
- * Note that this class should not depend on any classes or libraries except the JDK default runtime.
+ * This class is designed to be independent on any Ignite classes as possible.
+ * Please make sure to pass the path separator character as the 1st parameter to the main method.
*/
public class HadoopClasspathMain {
/**
+ * Main method to be executed from scripts. It prints the classpath to the standard output.
*
- * @param args
+ * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
*/
public static void main(String[] args) throws Exception {
- final char sep = File.pathSeparatorChar;
+ if (args.length < 1) {
+ err.println("Path separator must be passed as the 1st argument.");
- List<String> cp = getAsProcessClasspath();
+ System.exit(1);
+ }
+
+ final String sep = args[0];
+
+ List<String> cp = HadoopClasspathUtils.getAsProcessClasspath();
for (String s: cp) {
if (s != null && s.length() > 0) {
@@ -37,208 +36,4 @@ public class HadoopClasspathMain {
out.println();
}
-
- /**
- *
- * @return
- */
- private static List<String> getAsProcessClasspath() throws IOException {
- Collection<DirAndMask> dams = getClasspathBaseDirectories();
-
- List<String> list = new ArrayList<>(32);
-
- for (DirAndMask dam: dams)
- addAsJavaProcessClasspathElement(list, dam.file, dam.mask);
-
- // Sort the classpath elements to make it more reliable.
- Collections.sort(list);
-
- return list;
- }
-
- /**
- *
- * @return
- * @throws IOException
- */
- public static List<URL> getAsUrlList() throws IOException {
- Collection<DirAndMask> dams = getClasspathBaseDirectories();
-
- List<URL> list = new ArrayList<>(32);
-
- for (DirAndMask dam: dams)
- // Note that this procedure does not use '*' classpath patterns,
- // but adds all the children explicitly:
- addUrls(list, dam.file, dam.mask);
-
- Collections.sort(list, new Comparator<URL>() {
- @Override public int compare(URL u1, URL u2) {
- String s1 = String.valueOf(u1);
- String s2 = String.valueOf(u2);
-
- return s1.compareTo(s2);
- }
- });
-
- for (URL u: list)
- err.println(u);
-
- return list;
- }
-
- /**
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory: " + dir);
-
- for (File file : files)
- res.add(file.toURI().toURL());
- }
-
-
- /**
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
- */
- private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith) throws IOException {
- if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
- throw new IOException("Path is not an existing readable directory: " + dir);
-
- if (startsWith == null)
- res.add(dir.getAbsolutePath() + File.separator + '*');
- else {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory: " + dir);
-
- for (File file : files)
- res.add(file.getAbsolutePath());
- }
- }
-
- /**
- * @return HADOOP_HOME Variable.
- */
- public static String hadoopHome() {
- String prefix = getEnv("HADOOP_PREFIX", null);
-
- return getEnv("HADOOP_HOME", prefix);
- }
-
- /**
- *
- * @return
- * @throws FileNotFoundException
- */
- public static Collection<DirAndMask> getClasspathBaseDirectories() throws FileNotFoundException {
- final String hadoopHome = hadoopHome();
-
- String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
- String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
- String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
-
- Collection<DirAndMask> c = new ArrayList<>();
-
- c.add(new DirAndMask(new File(commonHome, "lib"), null));
- c.add(new DirAndMask(new File(hdfsHome, "lib"), null));
- c.add(new DirAndMask(new File(mapredHome, "lib"), null));
-
- c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
- c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
- c.add(new DirAndMask(new File(commonHome, "lib"), "hadoop-auth-"));
-
- c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
-
- c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-common"));
- c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-core"));
-
- return c;
- }
-
- public static class DirAndMask {
- DirAndMask(File f, String m) {
- file = f;
- mask = m;
- }
- public final File file;
- public final String mask;
- }
-
- /**
- * Checks if the variable is empty.
- *
- * @param envVarName Environment variable name.
- * @param hadoopHome The current value.
- * @param expHadoopHomeRelativePath The path relative to Hadoop home.
- * @throws FileNotFoundException If the value is empty.
- */
- private static String resolveLocation(String envVarName, String hadoopHome,
- String expHadoopHomeRelativePath) throws FileNotFoundException {
- String val = getEnv(envVarName, null);
-
- if (val == null) {
- // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!isExistingDirectory(hadoopHome))
- throw new FileNotFoundException("Failed to resolve Hadoop installation location. " +
- envVarName + " or HADOOP_HOME environment variable should be set.");
-
- val = hadoopHome + expHadoopHomeRelativePath;
- }
-
- if (!isExistingDirectory(val))
- throw new FileNotFoundException("Failed to resolve Hadoop location. [path=" + val + ']');
-
- err.println(envVarName + " resolved to " + val);
-
- return val;
- }
-
- /**
- * Note that this method does not treat empty value as an absent value.
- *
- * @param name Variable name.
- * @param dflt Default.
- * @return Value.
- */
- private static String getEnv(String name, String dflt) {
- String res = System.getProperty(name);
-
- if (res == null)
- res = System.getenv(name);
-
- return res == null ? dflt : res;
- }
-
- /**
- * Answers if the given path denotes existing directory.
- *
- * @param path The directory path.
- * @return 'true' if the given path denotes an existing directory.
- */
- private static boolean isExistingDirectory(String path) {
- if (path == null)
- return false;
-
- Path p = Paths.get(path);
-
- return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
new file mode 100644
index 0000000..c4a2fcc
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -0,0 +1,232 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static java.lang.System.err;
+
+/**
+ * Hadoop classpath utilities.
+ */
+public class HadoopClasspathUtils {
+ /**
+ * Gets Hadoop class path as list of classpath elements for process.
+ *
+ * @return List of the class path elements.
+ * @throws IOException On error.
+ */
+ static List<String> getAsProcessClasspath() throws IOException {
+ Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+ List<String> list = new ArrayList<>(32);
+
+ for (DirAndMask dam: dams)
+ addAsJavaProcessClasspathElement(list, dam.dir, dam.mask);
+
+ return list;
+ }
+
+ /**
+ * Gets Hadoop class path as a list of URLs (for in-process class loader usage).
+ *
+ * @return List of class path URLs.
+ * @throws IOException On error.
+ */
+ public static List<URL> getAsUrlList() throws IOException {
+ Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+ List<URL> list = new ArrayList<>(32);
+
+ for (DirAndMask dam: dams)
+ // Note that this procedure does not use '*' classpath patterns,
+ // but adds all the children explicitly:
+ addUrls(list, dam.dir, dam.mask);
+
+ return list;
+ }
+
+ /**
+ * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
+ *
+ * @param res Result.
+ * @param dir Directory.
+ * @param startsWith Starts with prefix.
+ * @throws MalformedURLException If failed.
+ */
+ private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return startsWith == null || name.startsWith(startsWith);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory. [dir=" + dir + ']');
+
+ for (File file : files)
+ res.add(file.toURI().toURL());
+ }
+
+
+ /**
+ * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
+ *
+ * @param res Result.
+ * @param dir Directory.
+ * @param startsWith Starts with prefix.
+ * @throws MalformedURLException If failed.
+ */
+ private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith) throws IOException {
+ if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
+ throw new IOException("Path is not an existing readable directory. [dir=" + dir + ']');
+
+ if (startsWith == null)
+ res.add(dir.getAbsolutePath() + File.separator + '*');
+ else {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith(startsWith);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory. [" + dir + ']');
+
+ for (File file : files)
+ res.add(file.getAbsolutePath());
+ }
+ }
+
+ /**
+ * @return HADOOP_HOME Variable.
+ */
+ public static String hadoopHome() {
+ String prefix = getEnv("HADOOP_PREFIX", null);
+
+ return getEnv("HADOOP_HOME", prefix);
+ }
+
+ /**
+ * Gets base directories to discover classpath elements in.
+ *
+ * @return Collection of directory and mask pairs.
+ * @throws FileNotFoundException if a mandatory classpath location is not found.
+ */
+ private static Collection<DirAndMask> getClasspathBaseDirectories() throws FileNotFoundException {
+ final String hadoopHome = hadoopHome();
+
+ String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
+ String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
+ String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+
+ Collection<DirAndMask> c = new ArrayList<>();
+
+ c.add(new DirAndMask(new File(commonHome, "lib"), null));
+ c.add(new DirAndMask(new File(hdfsHome, "lib"), null));
+ c.add(new DirAndMask(new File(mapredHome, "lib"), null));
+
+ c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
+ c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
+ c.add(new DirAndMask(new File(commonHome, "lib"), "hadoop-auth-"));
+
+ c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
+
+ c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-common"));
+ c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-core"));
+
+ return c;
+ }
+
+ /**
+ * Simple pair-like structure to hold directory name and a mask assigned to it.
+ */
+ public static class DirAndMask {
+ /**
+ * Constructor.
+ *
+ * @param dir The directory.
+ * @param mask The mask.
+ */
+ DirAndMask(File dir, String mask) {
+ this.dir = dir;
+ this.mask = mask;
+ }
+
+ /** The path. */
+ public final File dir;
+
+ /** The mask. */
+ public final String mask;
+ }
+
+ /**
+ * Checks if the variable is empty.
+ *
+ * @param envVarName Environment variable name.
+ * @param hadoopHome The current value.
+ * @param expHadoopHomeRelativePath The path relative to Hadoop home.
+ * @throws FileNotFoundException If the value is empty.
+ */
+ private static String resolveLocation(String envVarName, String hadoopHome,
+ String expHadoopHomeRelativePath) throws FileNotFoundException {
+ String val = getEnv(envVarName, null);
+
+ if (val == null) {
+ // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
+ if (!isExistingDirectory(hadoopHome))
+ throw new FileNotFoundException("Failed to resolve Hadoop installation location. " +
+ envVarName + " or HADOOP_HOME environment variable should be set.");
+
+ val = hadoopHome + expHadoopHomeRelativePath;
+ }
+
+ if (!isExistingDirectory(val))
+ throw new FileNotFoundException("Failed to resolve Hadoop location. [path=" + val + ']');
+
+ // Print diagnostic output:
+ err.println(envVarName + " resolved to " + val);
+
+ return val;
+ }
+
+ /**
+ * Note that this method does not treat empty value as an absent value.
+ *
+ * @param name Variable name.
+ * @param dflt Default.
+ * @return Value.
+ */
+ private static String getEnv(String name, String dflt) {
+ String res = System.getProperty(name);
+
+ if (res == null)
+ res = System.getenv(name);
+
+ return res == null ? dflt : res;
+ }
+
+ /**
+ * Answers if the given path denotes existing directory.
+ *
+ * @param path The directory path.
+ * @return 'true' if the given path denotes an existing directory.
+ */
+ private static boolean isExistingDirectory(String path) {
+ if (path == null)
+ return false;
+
+ Path p = Paths.get(path);
+
+ return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0d08f86/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index a502c3e..98e1fc5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -33,9 +33,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.hadoopHome;
-
/**
* Hadoop processor.
*/
@@ -74,13 +71,13 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
- if (hadoopHome() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + hadoopHome());
+ if (HadoopClasspathUtils.hadoopHome() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + HadoopClasspathUtils.hadoopHome());
boolean ok = false;
try { // Check for Hadoop installation.
- hadoopUrls();
+ HadoopClassLoader.hadoopUrls();
ok = true;
}
[08/27] ignite git commit: IGNITE-3184: minor corrections.
Posted by vo...@apache.org.
IGNITE-3184: minor corrections.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/39c06ea8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/39c06ea8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/39c06ea8
Branch: refs/heads/ignite-3185
Commit: 39c06ea83c677f098ee1ba1215fa0e33775be2cc
Parents: f7a7bc2
Author: iveselovskiy <iv...@gridgain.com>
Authored: Wed Jun 22 12:14:53 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Wed Jun 22 12:14:53 2016 +0300
----------------------------------------------------------------------
.../internal/processors/hadoop/HadoopClasspathUtils.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/39c06ea8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 9f6d6b1..e122913 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -169,12 +169,12 @@ public class HadoopClasspathUtils {
}
/**
- * Checks if the variable is empty.
+ * Resolves a Hadoop location directory.
*
- * @param envVarName Environment variable name.
- * @param hadoopHome The current value.
- * @param expHadoopHomeRelativePath The path relative to Hadoop home.
- * @throws FileNotFoundException If the value is empty.
+ * @param envVarName Environment variable name. The value denotes the location path.
+ * @param hadoopHome Hadoop home location, may be null.
+ * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
+ * @throws FileNotFoundException If the value cannot be resolved to an existing directory.
*/
private static String resolveLocation(String envVarName, String hadoopHome,
String expHadoopHomeRelativePath) throws FileNotFoundException {
[06/27] ignite git commit: IGNITE-3184: correction to read
/etc/default/hadoop
Posted by vo...@apache.org.
IGNITE-3184: correction to read /etc/default/hadoop
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/69101268
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/69101268
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/69101268
Branch: refs/heads/ignite-3185
Commit: 691012680e4d9e2b3002764d29b1096f93bb7aad
Parents: d27924c
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Jun 21 15:44:33 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Jun 21 15:44:33 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.sh | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/69101268/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 1f51b59..a85cba3 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -72,6 +72,18 @@ if [ "${USER_LIBS}" != "" ]; then
fi
if [ "${HADOOP_EDITION}" == "1" ]; then
+ # Resolve constants.
+ HADOOP_DEFAULTS="/etc/default/hadoop"
+
+ #
+ # Resolve the rest of Hadoop environment variables.
+ #
+ if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z "${HADOOP_MAPRED_HOME}" ]]; then
+ if [ -f "$HADOOP_DEFAULTS" ]; then
+ source "$HADOOP_DEFAULTS"
+ fi
+ fi
+
IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \
org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
[21/27] ignite git commit: Inlined "isEmpty(String)" method.
Posted by vo...@apache.org.
Inlined "isEmpty(String)" method.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/75601f86
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/75601f86
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/75601f86
Branch: refs/heads/ignite-3185
Commit: 75601f86ae6486e231b95271bf9f8fb5daea9ac2
Parents: 97361b3
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 13:15:37 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 13:15:37 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/75601f86/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 632953f..c83231e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -17,8 +17,6 @@
package org.apache.ignite.internal.processors.hadoop;
-import org.apache.ignite.internal.util.typedef.F;
-
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -111,7 +109,7 @@ public class HadoopClasspathUtils {
String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
// If any composite location is defined, use only them.
- if (!F.isEmpty(commonHome) || !F.isEmpty(hdfsHome) || !F.isEmpty(mapredHome)) {
+ if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) {
HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
if (res.valid())
@@ -123,7 +121,7 @@ public class HadoopClasspathUtils {
"[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
"[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
}
- else if (!F.isEmpty(hadoopHome)) {
+ else if (!isEmpty(hadoopHome)) {
// All further checks will be based on HADOOP_HOME, so check for it's existence.
if (!exists(hadoopHome))
throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
@@ -225,6 +223,16 @@ public class HadoopClasspathUtils {
}
/**
+ * Check if string is empty.
+ *
+ * @param val Value.
+ * @return {@code True} if empty.
+ */
+ private static boolean isEmpty(String val) {
+ return val == null || val.isEmpty();
+ }
+
+ /**
* Simple pair-like structure to hold directory name and a mask assigned to it.
*/
private static class SearchDirectory {
[19/27] ignite git commit: IGNITE-3185: corrections after merge.
Posted by vo...@apache.org.
IGNITE-3185: corrections after merge.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c2769351
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c2769351
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c2769351
Branch: refs/heads/ignite-3185
Commit: c27693514417a4f7793414ba272522189a19ca55
Parents: 0bd836a
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Jun 27 19:30:01 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Jun 27 19:30:01 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 17 ++++++++++-------
2 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c2769351/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 6c1f178..610be50 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -237,7 +237,7 @@ public class HadoopClasspathUtils {
* @param path The directory path.
* @return {@code True} if the given path denotes an existing directory.
*/
- private static boolean directoryExists(String path) {
+ static boolean directoryExists(String path) {
if (path == null)
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c2769351/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index ebd085a..ee2c659 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -34,7 +34,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
/**
* Hadoop processor.
*/
@@ -77,11 +76,15 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
HadoopLocations loc = HadoopClasspathUtils.getHadoopLocations();
if (loc.home() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
-
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+ U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_HOME.toString() + " is set to "
+ + loc.home());
+
+ U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_COMMON_HOME.toString() + " is set to "
+ + loc.commonHome());
+ U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_HDFS_HOME.toString() + " is set to "
+ + loc.hdfsHome());
+ U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_MAPRED_HOME.toString() + " is set to "
+ + loc.mapredHome());
}
catch (IOException ioe) {
throw new IgniteCheckedException(ioe);
@@ -165,7 +168,7 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
@Override public Hadoop hadoop() {
if (hadoop == null)
throw new IllegalStateException("Hadoop accelerator is disabled (Hadoop is not in classpath, " +
- "is HADOOP_HOME environment variable set?)");
+ "are Hadoop location environment variables set?)");
return hadoop;
}
[16/27] ignite git commit: IGNITE-3185:
Posted by vo...@apache.org.
IGNITE-3185:
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e575bf54
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e575bf54
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e575bf54
Branch: refs/heads/ignite-3185
Commit: e575bf5416a526918b30e3ee445ef2b8d9bc6eab
Parents: 04f2434
Author: iveselovskiy <iv...@gridgain.com>
Authored: Fri Jun 24 19:20:32 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Fri Jun 24 19:20:32 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 180 ++-----------------
.../processors/hadoop/HadoopLocations.java | 53 +++++-
2 files changed, 70 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e575bf54/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 1a62a88..598c1c3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -77,113 +77,13 @@ public class HadoopClasspathUtils {
return res;
}
-// /**
-// * @return HADOOP_HOME Variable.
-// */
-// private static String hadoopHome() {
-// String prefix = getEnv("HADOOP_PREFIX", null);
-//
-// return getEnv("HADOOP_HOME", prefix);
-// }
-
-// /**
-// * Simple structure to hold Hadoop directory locations.
-// */
-// public static class HadoopLocations {
-// /** HADOOP_HOME, may be null. */
-// public final String home;
-// /** HADOOP_COMMON_HOME */
-// public final String common;
-// /** HADOOP_HDFS_HOME */
-// public final String hdfs;
-// /** HADOOP_MAPRED_HOME */
-// public final String mapred;
-//
-// /**
-// * Constructor.
-// *
-// * @param home HADOOP_HOME
-// * @param common HADOOP_COMMON_HOME
-// * @param hdfs HADOOP_HDFS_HOME
-// * @param mapred HADOOP_MAPRED_HOME
-// */
-// HadoopLocations(String home, String common, String hdfs, String mapred) {
-// this.home = home;
-// this.common = common;
-// this.hdfs = hdfs;
-// this.mapred = mapred;
-// }
-// }
-
/**
- * Simple structure to hold Hadoop directory locations.
+ * @return HADOOP_HOME Variable.
*/
- public static class HadoopLocations {
- /** HADOOP_HOME, may be null. */
- public final String home;
- /** HADOOP_COMMON_HOME */
- public final String common;
- /** HADOOP_HDFS_HOME */
- public final String hdfs;
- /** HADOOP_MAPRED_HOME */
- public final String mapred;
-
- /**
- * Constructor.
- *
- * @param home HADOOP_HOME
- * @param common HADOOP_COMMON_HOME
- * @param hdfs HADOOP_HDFS_HOME
- * @param mapred HADOOP_MAPRED_HOME
- */
- HadoopLocations(String home, String common, String hdfs, String mapred) {
- this.home = home;
- this.common = common;
- this.hdfs = hdfs;
- this.mapred = mapred;
- }
-
- /**
- * Answers if all the base directories are defined.
- *
- * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
- */
- public boolean isDefined() {
- return common != null && hdfs != null && mapred != null;
- }
-
- /**
- * Answers if all the base directories exist.
- *
- * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
- */
- public boolean exists() {
- return isExistingDirectory(common)
- && isExistingDirectory(hdfs)
- && isExistingDirectory(mapred);
- }
+ private static String hadoopHome() {
+ String prefix = systemOrEnv("HADOOP_PREFIX", null);
- /**
- * Checks if all the base directories exist.
- *
- * @return this reference.
- * @throws IOException if any of the base directories does not exist.
- */
- public HadoopLocations existsOrException() throws IOException {
- if (!isExistingDirectory(common))
- throw new IOException("Failed to resolve Hadoop installation location. HADOOP_COMMON_HOME " +
- "or HADOOP_HOME environment variable should be set.");
-
- if (!isExistingDirectory(hdfs))
- throw new IOException("Failed to resolve Hadoop installation location. HADOOP_HDFS_HOME " +
- "or HADOOP_HOME environment variable should be set.");
-
- if (!isExistingDirectory(mapred))
- throw new IOException("Failed to resolve Hadoop installation location. HADOOP_MAPRED_HOME " +
- "or HADOOP_HOME environment variable should be set.");
-
- return this;
- }
+ return systemOrEnv("HADOOP_HOME", prefix);
}
/**
@@ -194,9 +94,9 @@ public class HadoopClasspathUtils {
private static HadoopLocations getEnvHadoopLocations() {
return new HadoopLocations(
hadoopHome(),
- getEnv("HADOOP_COMMON_HOME", null),
- getEnv("HADOOP_HDFS_HOME", null),
- getEnv("HADOOP_MAPRED_HOME", null)
+ systemOrEnv("HADOOP_COMMON_HOME", null),
+ systemOrEnv("HADOOP_HDFS_HOME", null),
+ systemOrEnv("HADOOP_MAPRED_HOME", null)
);
}
@@ -231,18 +131,23 @@ public class HadoopClasspathUtils {
hadoopHome + "/../hadoop-mapreduce-client/");
}
-
/**
* Gets Hadoop locations.
*
* @return The locations as determined from the environment.
*/
- public static HadoopLocations getHadoopLocations() throws IOException {
+ private static HadoopLocations getHadoopLocations() throws IOException {
+ // 1. Get Hadoop locations from the environment:
+ HadoopLocations loc = getEnvHadoopLocations();
+
+ if (loc.isDefined())
+ return loc.existsOrException();
+
final String hadoopHome = hadoopHome();
if (hadoopHome != null) {
// If home is defined, it must exist:
- if (!isExistingDirectory(hadoopHome))
+ if (!directoryExists(hadoopHome))
throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
+ hadoopHome + ']');
@@ -269,7 +174,7 @@ public class HadoopClasspathUtils {
* @throws IOException if a mandatory classpath location is not found.
*/
private static Collection<SearchDirectory> classpathDirectories() throws IOException {
- HadoopLocations loc = hadoopLocations();
+ HadoopLocations loc = getHadoopLocations();
Collection<SearchDirectory> res = new ArrayList<>();
@@ -288,55 +193,6 @@ public class HadoopClasspathUtils {
return res;
}
-// /**
-// * Simple pair-like structure to hold directory name and a mask assigned to it.
-// */
-// public static class DirAndMask {
-// /**
-// * Constructor.
-// *
-// * @param dir The directory.
-// * @param mask The mask.
-// */
-// DirAndMask(File dir, String mask) {
-// this.dir = dir;
-// this.mask = mask;
-// }
-//
-// /** The path. */
-// public final File dir;
-//
-// /** The mask. */
-// public final String mask;
-// }
-
- /**
- * Resolves a Hadoop location directory.
- *
- * @param envVarName Environment variable name. The value denotes the location path.
- * @param hadoopHome Hadoop home location, may be null.
- * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
- * @throws IOException If the value cannot be resolved to an existing directory.
- */
- private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
- throws IOException {
- String val = getEnv(envVarName, null);
-
- if (val == null) {
- // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!isExistingDirectory(hadoopHome))
- throw new IOException("Failed to resolve Hadoop installation location. " +
- envVarName + " or HADOOP_HOME environment variable should be set.");
-
- val = hadoopHome + expHadoopHomeRelativePath;
- }
-
- if (!isExistingDirectory(val))
- throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
-
- return val;
- }
-
/**
* Note that this method does not treat empty value as an absent value.
*
@@ -359,7 +215,7 @@ public class HadoopClasspathUtils {
* @param path The directory path.
* @return {@code True} if the given path denotes an existing directory.
*/
- private static boolean directoryExists(String path) {
+ static boolean directoryExists(String path) {
if (path == null)
return false;
@@ -415,4 +271,4 @@ public class HadoopClasspathUtils {
return filter != null;
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e575bf54/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
index b3e2d12..597a8dc 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.processors.hadoop;
+import java.io.*;
+
/**
* Simple structure to hold Hadoop directory locations.
*/
@@ -75,4 +77,53 @@ public class HadoopLocations {
public String mapredHome() {
return mapredHome;
}
-}
+
+ /**
+ * Answers if all the base directories are defined.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+ */
+ public boolean isDefined() {
+ return commonHome != null && hdfsHome != null && mapredHome != null;
+ }
+
+ /**
+ * Answers if all the base directories exist.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+ */
+ public boolean exists() {
+ return HadoopClasspathUtils.directoryExists(commonHome) && HadoopClasspathUtils.directoryExists(hdfsHome)
+ && HadoopClasspathUtils.directoryExists(mapredHome);
+ }
+
+ /**
+ * Checks if all the base directories exist.
+ *
+ * @return this reference.
+ * @throws IOException if any of the base directories does not exist.
+ */
+ public HadoopLocations existsOrException() throws IOException {
+ if (!HadoopClasspathUtils.directoryExists(commonHome))
+ throw ioe("HADOOP_COMMON_HOME");
+
+ if (!HadoopClasspathUtils.directoryExists(hdfsHome))
+ throw ioe("HADOOP_HDFS_HOME");
+
+ if (!HadoopClasspathUtils.directoryExists(mapredHome))
+ throw ioe("HADOOP_MAPRED_HOME");
+
+ return this;
+ }
+
+ /**
+ * Constructs the exception.
+ *
+ * @param var The 1st variable name to mention in the exception text.
+ * @return The exception.
+ */
+ private IOException ioe(String var) {
+ return new IOException("Failed to resolve Hadoop installation location. " + var +
+ " or HADOOP_HOME environment variable should be set.");
+ }
+}
\ No newline at end of file
[20/27] ignite git commit: IGNITE-3185: Review.
Posted by vo...@apache.org.
IGNITE-3185: Review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/97361b3b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/97361b3b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/97361b3b
Branch: refs/heads/ignite-3185
Commit: 97361b3bf09beb64e3b2bd566d31621117980763
Parents: c276935
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 11:06:51 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 11:06:51 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 223 ++++++++-----------
.../processors/hadoop/HadoopLocations.java | 94 ++++----
.../processors/hadoop/HadoopProcessor.java | 20 +-
3 files changed, 148 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/97361b3b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 610be50..632953f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.processors.hadoop;
+import org.apache.ignite.internal.util.typedef.F;
+
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -33,6 +35,24 @@ import java.util.List;
* Hadoop classpath utilities.
*/
public class HadoopClasspathUtils {
+ /** Prefix directory. */
+ public static final String PREFIX = "HADOOP_PREFIX";
+
+ /** Home directory. */
+ public static final String HOME = "HADOOP_HOME";
+
+ /** Home directory. */
+ public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
+
+ /** Home directory. */
+ public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
+
+ /** Home directory. */
+ public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
+
+ /** Empty string. */
+ private static final String EMPTY_STR = "";
+
/**
* Gets Hadoop class path as list of classpath elements for process.
*
@@ -78,115 +98,73 @@ public class HadoopClasspathUtils {
}
/**
- * @return HADOOP_HOME Variable.
- */
- private static String hadoopHome() {
- String prefix = systemOrEnv("HADOOP_PREFIX", null);
-
- return systemOrEnv(HadoopVariable.HADOOP_HOME.toString(), prefix);
- }
-
- /**
- * Gets locations from the environment.
- *
- * @return The locations as determined from the environment.
- */
- private static HadoopLocations getEnvHadoopLocations() {
- return new HadoopLocations(
- hadoopHome(),
- systemOrEnv(HadoopVariable.HADOOP_COMMON_HOME.toString(), null),
- systemOrEnv(HadoopVariable.HADOOP_HDFS_HOME.toString(), null),
- systemOrEnv(HadoopVariable.HADOOP_MAPRED_HOME.toString(), null)
- );
- }
-
- /**
- * Gets locations assuming Apache Hadoop distribution layout.
- *
- * @return The locations as for Apache distribution.
- */
- private static HadoopLocations getApacheHadoopLocations(String hadoopHome) {
- return new HadoopLocations(hadoopHome,
- hadoopHome + "/share/hadoop/common",
- hadoopHome + "/share/hadoop/hdfs",
- hadoopHome + "/share/hadoop/mapreduce");
- }
-
- /** HDP Hadoop locations. */
- private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
- "/usr/hdp/current/hadoop-client",
- "/usr/hdp/current/hadoop-client",
- "/usr/hdp/current/hadoop-hdfs-client",
- "/usr/hdp/current/hadoop-mapreduce-client");
-
- /**
- * HDP locations relative to an arbitrary Hadoop home.
- *
- * @param hadoopHome The hadoop home.
- * @return The locations.
- */
- private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
- return new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs-client",
- hadoopHome + "/../hadoop-mapreduce-client");
- }
-
- /**
* Gets Hadoop locations.
*
* @return The locations as determined from the environment.
*/
- static HadoopLocations getHadoopLocations() throws IOException {
- // 1. Get Hadoop locations from the environment:
- HadoopLocations loc = getEnvHadoopLocations();
-
- if (loc.isDefined())
- return loc.existsOrException();
-
- final String hadoopHome = hadoopHome();
-
- if (hadoopHome != null) {
- // If home is defined, it must exist:
- if (!directoryExists(hadoopHome))
- throw ioeNotFound(HadoopVariable.HADOOP_HOME.toString(), hadoopHome);
-
- // 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
- loc = getApacheHadoopLocations(hadoopHome);
+ public static HadoopLocations locations() throws IOException {
+ // Query environment.
+ String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR));
- if (loc.exists())
- return loc;
+ String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR);
+ String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR);
+ String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
- // 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
- loc = getHdpLocationsRelative(hadoopHome);
+ // If any composite location is defined, use only them.
+ if (!F.isEmpty(commonHome) || !F.isEmpty(hdfsHome) || !F.isEmpty(mapredHome)) {
+ HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
- if (loc.exists())
- return loc;
-
- throw new IOException("Hadoop installation locations not found based on defined "
- + HadoopVariable.HADOOP_HOME.toString() + " value. [" + HadoopVariable.HADOOP_HOME.toString()
- + "=" + hadoopHome + ']');
+ if (res.valid())
+ return res;
+ else
+ throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " +
+ "either undefined or point to nonexistent directories [" +
+ "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " +
+ "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
+ "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
}
+ else if (!F.isEmpty(hadoopHome)) {
+ // All further checks will be based on HADOOP_HOME, so check for it's existence.
+ if (!exists(hadoopHome))
+ throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
+ "variable points to nonexistent directory: " + hadoopHome);
+
+ // Create the list of probes and then iterate over them.
+ List<HadoopLocations> probes = new ArrayList<>();
+
+ // Probe for Apache Hadoop.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs", hadoopHome + "/share/hadoop/mapreduce"));
+
+ // Probe for CDH with normal HADOOP_HOME directory.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop",
+ hadoopHome + "/hadoop-hdfs", hadoopHome + "/hadoop-mapreduce"));
+
+ // Probe for CDH with HADOOP_HOME equal to HADOOP_COMMON_HOME.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs", hadoopHome + "/../hadoop-mapreduce"));
+
+ // Probe for HDP with normal HADOOP_HOME directory.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop-client,",
+ hadoopHome + "/hadoop-hdfs-client", hadoopHome + "/hadoop-mapreduce-client"));
+
+ // Probe for HDP with HADOOP_HOME equal to HADOOP_COMMON_HOME.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client", hadoopHome + "/../hadoop-mapreduce-client"));
+
+ for (HadoopLocations probe : probes) {
+ if (probe.valid())
+ return probe;
+ }
- // 4. Try absolute HDP (Hortonworks) location:
- if (HDP_HADOOP_LOCATIONS.exists())
- return HDP_HADOOP_LOCATIONS;
-
- throw new IOException("Hadoop installation locations cannot be found. Please define environment variables "
- + HadoopVariable.HADOOP_COMMON_HOME.toString() + ", "
- + HadoopVariable.HADOOP_HDFS_HOME.toString() + ", "
- + HadoopVariable.HADOOP_MAPRED_HOME.toString() + ", or "
- + HadoopVariable.HADOOP_HOME.toString() + ".");
- }
-
- /**
- * Convenience method to construct Exception.
- *
- * @param var The variable name.
- * @param dir The dir.
- * @return The Exception.
- */
- static IOException ioeNotFound(String var, String dir) {
- return new IOException(var + " location is not an existing readable directory. [" + var + "=" + dir + ']');
+ throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
+ "points to invalid directory: " + hadoopHome);
+ }
+ else {
+ // Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
+ throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " +
+ "variable and point it to your Hadoop distribution).");
+ }
}
/**
@@ -196,30 +174,30 @@ public class HadoopClasspathUtils {
* @throws IOException if a mandatory classpath location is not found.
*/
private static Collection<SearchDirectory> classpathDirectories() throws IOException {
- HadoopLocations loc = getHadoopLocations();
+ HadoopLocations loc = locations();
Collection<SearchDirectory> res = new ArrayList<>();
- res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null));
- res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null));
- res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.common(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.mapred(), "lib"), null));
- res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-"));
- res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-"));
+ res.add(new SearchDirectory(new File(loc.common()), "hadoop-common-"));
+ res.add(new SearchDirectory(new File(loc.common()), "hadoop-auth-"));
- res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-"));
+ res.add(new SearchDirectory(new File(loc.hdfs()), "hadoop-hdfs-"));
- res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common"));
- res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core"));
+ res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-common"));
+ res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-core"));
return res;
}
/**
- * Note that this method does not treat empty value as an absent value.
+ * Get system property or environment variable with the given name.
*
* @param name Variable name.
- * @param dflt Default.
+ * @param dflt Default value.
* @return Value.
*/
private static String systemOrEnv(String name, String dflt) {
@@ -228,7 +206,7 @@ public class HadoopClasspathUtils {
if (res == null)
res = System.getenv(name);
- return res == null ? dflt : res;
+ return res != null ? res : dflt;
}
/**
@@ -237,7 +215,7 @@ public class HadoopClasspathUtils {
* @param path The directory path.
* @return {@code True} if the given path denotes an existing directory.
*/
- static boolean directoryExists(String path) {
+ public static boolean exists(String path) {
if (path == null)
return false;
@@ -266,7 +244,7 @@ public class HadoopClasspathUtils {
this.dir = dir;
this.filter = filter;
- if (!directoryExists(dir.getAbsolutePath()))
+ if (!exists(dir.getAbsolutePath()))
throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
}
@@ -300,21 +278,4 @@ public class HadoopClasspathUtils {
return filter != null;
}
}
-
- /**
- * Enumeration of commonly used Hadoop environment variables.
- */
- public enum HadoopVariable {
- /** Variable HADOOP_HOME. */
- HADOOP_HOME,
-
- /** Variable HADOOP_COMMON_HOME. */
- HADOOP_COMMON_HOME,
-
- /** Variable HADOOP_HDFS_HOME. */
- HADOOP_HDFS_HOME,
-
- /** Variable HADOOP_MAPRED_HOME. */
- HADOOP_MAPRED_HOME,
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/97361b3b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
index 9be42a5..a90007f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -17,8 +17,6 @@
package org.apache.ignite.internal.processors.hadoop;
-import java.io.*;
-
/**
* Simple structure to hold Hadoop directory locations.
*/
@@ -27,27 +25,42 @@ public class HadoopLocations {
private final String home;
/** Common home. */
- private final String commonHome;
+ private final String common;
/** HDFS home. */
- private final String hdfsHome;
+ private final String hdfs;
/** Mapred home. */
- private final String mapredHome;
+ private final String mapred;
+
+ /** Whether common home exists. */
+ private final boolean commonExists;
+
+ /** Whether HDFS home exists. */
+ private final boolean hdfsExists;
+
+ /** Whether mapred home exists. */
+ private final boolean mapredExists;
/**
* Constructor.
*
* @param home Hadoop home.
- * @param commonHome Common home.
- * @param hdfsHome HDFS home.
- * @param mapredHome Mapred home.
+ * @param common Common home.
+ * @param hdfs HDFS home.
+ * @param mapred Mapred home.
*/
- public HadoopLocations(String home, String commonHome, String hdfsHome, String mapredHome) {
+ public HadoopLocations(String home, String common, String hdfs, String mapred) {
+ assert common != null && hdfs != null && mapred != null;
+
this.home = home;
- this.commonHome = commonHome;
- this.hdfsHome = hdfsHome;
- this.mapredHome = mapredHome;
+ this.common = common;
+ this.hdfs = hdfs;
+ this.mapred = mapred;
+
+ commonExists = HadoopClasspathUtils.exists(common);
+ hdfsExists = HadoopClasspathUtils.exists(hdfs);
+ mapredExists = HadoopClasspathUtils.exists(mapred);
}
/**
@@ -60,62 +73,51 @@ public class HadoopLocations {
/**
* @return Common home.
*/
- public String commonHome() {
- return commonHome;
+ public String common() {
+ return common;
}
/**
* @return HDFS home.
*/
- public String hdfsHome() {
- return hdfsHome;
+ public String hdfs() {
+ return hdfs;
}
/**
* @return Mapred home.
*/
- public String mapredHome() {
- return mapredHome;
+ public String mapred() {
+ return mapred;
}
/**
- * Answers if all the base directories are defined.
- *
- * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+ * @return Whether common home exists.
*/
- public boolean isDefined() {
- return commonHome != null && hdfsHome != null && mapredHome != null;
+ public boolean commonExists() {
+ return commonExists;
}
/**
- * Answers if all the base directories exist.
- *
- * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+ * @return Whether HDFS home exists.
*/
- public boolean exists() {
- return HadoopClasspathUtils.directoryExists(commonHome) && HadoopClasspathUtils.directoryExists(hdfsHome)
- && HadoopClasspathUtils.directoryExists(mapredHome);
+ public boolean hdfsExists() {
+ return hdfsExists;
}
/**
- * Checks if all the base directories exist.
- *
- * @return this reference.
- * @throws IOException if any of the base directories does not exist.
+ * @return Whether mapred home exists.
*/
- public HadoopLocations existsOrException() throws IOException {
- if (!HadoopClasspathUtils.directoryExists(commonHome))
- throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_COMMON_HOME.toString(),
- commonHome);
-
- if (!HadoopClasspathUtils.directoryExists(hdfsHome))
- throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_HDFS_HOME.toString(),
- hdfsHome);
-
- if (!HadoopClasspathUtils.directoryExists(mapredHome))
- throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_MAPRED_HOME.toString(),
- mapredHome);
+ public boolean mapredExists() {
+ return mapredExists;
+ }
- return this;
+ /**
+ * Whether all required directories exists.
+ *
+ * @return {@code True} if exists.
+ */
+ public boolean valid() {
+ return commonExists && hdfsExists && mapredExists;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/97361b3b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index ee2c659..bbf6e8e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffle;
import org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopEmbeddedTaskExecutor;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -73,18 +74,13 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
try {
- HadoopLocations loc = HadoopClasspathUtils.getHadoopLocations();
-
- if (loc.home() != null)
- U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_HOME.toString() + " is set to "
- + loc.home());
-
- U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_COMMON_HOME.toString() + " is set to "
- + loc.commonHome());
- U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_HDFS_HOME.toString() + " is set to "
- + loc.hdfsHome());
- U.quietAndInfo(log, HadoopClasspathUtils.HadoopVariable.HADOOP_MAPRED_HOME.toString() + " is set to "
- + loc.mapredHome());
+ HadoopLocations loc = HadoopClasspathUtils.locations();
+
+ if (!F.isEmpty(loc.home()))
+ U.quietAndInfo(log, HadoopClasspathUtils.HOME + " is set to " + loc.home());
+
+ U.quietAndInfo(log, "Resolved Hadoop classpath locations: " + loc.common() + ", " + loc.hdfs() + ", " +
+ loc.mapred());
}
catch (IOException ioe) {
throw new IgniteCheckedException(ioe);
[26/27] ignite git commit: IGNITE-3185: Minors.
Posted by vo...@apache.org.
IGNITE-3185: Minors.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c16c945c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c16c945c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c16c945c
Branch: refs/heads/ignite-3185
Commit: c16c945c06a37cf3120a87e50d6d1639967194bf
Parents: d7aa255
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Jul 1 15:06:52 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Jul 1 15:06:52 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/processors/hadoop/HadoopProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c16c945c/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index 8677c55..364e09b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -202,7 +202,7 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
loc.mapred());
}
catch (IOException ioe) {
- throw new IgniteCheckedException(ioe);
+ throw new IgniteCheckedException(ioe.getMessage(), ioe);
}
HadoopClassLoader.hadoopUrls();
[13/27] ignite git commit: IGNITE-3184: Final review.
Posted by vo...@apache.org.
IGNITE-3184: Final review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/52841f90
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/52841f90
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/52841f90
Branch: refs/heads/ignite-3185
Commit: 52841f90a132d5c91a66ae171cb0169663f04049
Parents: 9cc1607
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Jun 24 17:22:43 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Jun 24 17:22:43 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopClasspathMain.java | 16 +-
.../processors/hadoop/HadoopClasspathUtils.java | 242 +++++++------------
.../processors/hadoop/HadoopLocations.java | 78 ++++++
.../processors/hadoop/HadoopProcessor.java | 13 +-
5 files changed, 181 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/52841f90/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index c0251eb..1c844c4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -498,7 +498,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return hadoopUrls;
try {
- hadoopUrls = HadoopClasspathUtils.getAsUrlList();
+ hadoopUrls = HadoopClasspathUtils.classpathUrls();
}
catch (IOException e) {
throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/52841f90/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
index b1422d4..5279b7d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -17,8 +17,6 @@
package org.apache.ignite.internal.processors.hadoop;
-import java.util.List;
-
/**
* Main class to compose Hadoop classpath depending on the environment.
* This class is designed to be independent on any Ignite classes as possible.
@@ -32,17 +30,15 @@ public class HadoopClasspathMain {
*/
public static void main(String[] args) throws Exception {
if (args.length < 1)
- throw new IllegalArgumentException("Path separator must be passed as the 1st argument.");
+ throw new IllegalArgumentException("Path separator must be passed as the first argument.");
- final String sep = args[0];
+ String separator = args[0];
- List<String> cp = HadoopClasspathUtils.getAsProcessClasspath();
+ StringBuilder sb = new StringBuilder();
- for (String s: cp) {
- System.out.print(s);
- System.out.print(sep);
- }
+ for (String path : HadoopClasspathUtils.classpathForJavaProcess())
+ sb.append(path).append(separator);
- System.out.println();
+ System.out.println(sb);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/52841f90/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index bb7ccd3..a934f61 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -39,15 +39,19 @@ public class HadoopClasspathUtils {
* @return List of the class path elements.
* @throws IOException If failed.
*/
- static List<String> getAsProcessClasspath() throws IOException {
- Collection<DirAndMask> dams = getClasspathBaseDirectories();
+ public static List<String> classpathForJavaProcess() throws IOException {
+ List<String> res = new ArrayList<>();
- List<String> list = new ArrayList<>(32);
-
- for (DirAndMask dam: dams)
- addAsJavaProcessClasspathElement(list, dam.dir, dam.mask);
+ for (final SearchDirectory dir : classpathDirectories()) {
+ if (dir.hasFilter()) {
+ for (File file : dir.files())
+ res.add(file.getAbsolutePath());
+ }
+ else
+ res.add(dir.dir.getAbsolutePath() + File.separator + '*');
+ }
- return list;
+ return res;
}
/**
@@ -56,113 +60,21 @@ public class HadoopClasspathUtils {
* @return List of class path URLs.
* @throws IOException If failed.
*/
- public static List<URL> getAsUrlList() throws IOException {
- Collection<DirAndMask> dams = getClasspathBaseDirectories();
-
- List<URL> list = new ArrayList<>(32);
-
- for (DirAndMask dam: dams)
- // Note that this procedure does not use '*' classpath patterns,
- // but adds all the children explicitly:
- addUrls(list, dam.dir, dam.mask);
-
- return list;
- }
-
- /**
- * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
- *
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws IOException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory. [dir=" + dir + ']');
-
- for (File file : files) {
- try {
- res.add(file.toURI().toURL());
- }
- catch (MalformedURLException e) {
- throw new IOException("Failed to convert file path to URL: " + file.getPath());
- }
- }
- }
-
- /**
- * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
- *
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws IOException If failed.
- */
- private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith)
- throws IOException {
- if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
- throw new IOException("Path is not an existing readable directory. [dir=" + dir + ']');
+ public static List<URL> classpathUrls() throws IOException {
+ List<URL> res = new ArrayList<>();
- if (startsWith == null)
- res.add(dir.getAbsolutePath() + File.separator + '*');
- else {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith(startsWith);
+ for (SearchDirectory dir : classpathDirectories()) {
+ for (File file : dir.files()) {
+ try {
+ res.add(file.toURI().toURL());
}
- });
-
- if (files == null)
- throw new IOException("Path is not a directory. [" + dir + ']');
-
- for (File file : files)
- res.add(file.getAbsolutePath());
+ catch (MalformedURLException e) {
+ throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ }
+ }
}
- }
- /**
- * @return HADOOP_HOME Variable.
- */
- private static String hadoopHome() {
- String prefix = getEnv("HADOOP_PREFIX", null);
-
- return getEnv("HADOOP_HOME", prefix);
- }
-
- /**
- * Simple structure to hold Hadoop directory locations.
- */
- public static class HadoopLocations {
- /** HADOOP_HOME, may be null. */
- public final String home;
- /** HADOOP_COMMON_HOME */
- public final String common;
- /** HADOOP_HDFS_HOME */
- public final String hdfs;
- /** HADOOP_MAPRED_HOME */
- public final String mapred;
-
- /**
- * Constructor.
- *
- * @param home HADOOP_HOME
- * @param common HADOOP_COMMON_HOME
- * @param hdfs HADOOP_HDFS_HOME
- * @param mapred HADOOP_MAPRED_HOME
- */
- HadoopLocations(String home, String common, String hdfs, String mapred) {
- this.home = home;
- this.common = common;
- this.hdfs = hdfs;
- this.mapred = mapred;
- }
+ return res;
}
/**
@@ -170,8 +82,8 @@ public class HadoopClasspathUtils {
*
* @return The Hadoop locations, never null.
*/
- public static HadoopLocations getHadoopLocations() throws IOException {
- final String hadoopHome = hadoopHome();
+ public static HadoopLocations hadoopLocations() throws IOException {
+ final String hadoopHome = systemOrEnv("HADOOP_HOME", systemOrEnv("HADOOP_PREFIX", null));
String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
@@ -186,46 +98,24 @@ public class HadoopClasspathUtils {
* @return Collection of directory and mask pairs.
* @throws IOException if a mandatory classpath location is not found.
*/
- private static Collection<DirAndMask> getClasspathBaseDirectories() throws IOException {
- HadoopLocations loc = getHadoopLocations();
+ private static Collection<SearchDirectory> classpathDirectories() throws IOException {
+ HadoopLocations loc = hadoopLocations();
- Collection<DirAndMask> c = new ArrayList<>();
+ Collection<SearchDirectory> res = new ArrayList<>();
- c.add(new DirAndMask(new File(loc.common, "lib"), null));
- c.add(new DirAndMask(new File(loc.hdfs, "lib"), null));
- c.add(new DirAndMask(new File(loc.mapred, "lib"), null));
+ res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null));
- c.add(new DirAndMask(new File(loc.common), "hadoop-common-"));
- c.add(new DirAndMask(new File(loc.common), "hadoop-auth-"));
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-"));
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-"));
- c.add(new DirAndMask(new File(loc.hdfs), "hadoop-hdfs-"));
+ res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-"));
- c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-common"));
- c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-core"));
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common"));
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core"));
- return c;
- }
-
- /**
- * Simple pair-like structure to hold directory name and a mask assigned to it.
- */
- public static class DirAndMask {
- /**
- * Constructor.
- *
- * @param dir The directory.
- * @param mask The mask.
- */
- DirAndMask(File dir, String mask) {
- this.dir = dir;
- this.mask = mask;
- }
-
- /** The path. */
- public final File dir;
-
- /** The mask. */
- public final String mask;
+ return res;
}
/**
@@ -238,18 +128,18 @@ public class HadoopClasspathUtils {
*/
private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
throws IOException {
- String val = getEnv(envVarName, null);
+ String val = systemOrEnv(envVarName, null);
if (val == null) {
// The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!isExistingDirectory(hadoopHome))
+ if (!directoryExists(hadoopHome))
throw new IOException("Failed to resolve Hadoop installation location. " +
envVarName + " or HADOOP_HOME environment variable should be set.");
val = hadoopHome + expHadoopHomeRelativePath;
}
- if (!isExistingDirectory(val))
+ if (!directoryExists(val))
throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
return val;
@@ -262,7 +152,7 @@ public class HadoopClasspathUtils {
* @param dflt Default.
* @return Value.
*/
- private static String getEnv(String name, String dflt) {
+ private static String systemOrEnv(String name, String dflt) {
String res = System.getProperty(name);
if (res == null)
@@ -275,9 +165,9 @@ public class HadoopClasspathUtils {
* Answers if the given path denotes existing directory.
*
* @param path The directory path.
- * @return 'true' if the given path denotes an existing directory.
+ * @return {@code True} if the given path denotes an existing directory.
*/
- private static boolean isExistingDirectory(String path) {
+ private static boolean directoryExists(String path) {
if (path == null)
return false;
@@ -285,4 +175,52 @@ public class HadoopClasspathUtils {
return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
}
+
+ /**
+ * Simple pair-like structure to hold directory name and a mask assigned to it.
+ */
+ public static class SearchDirectory {
+ /** File. */
+ private final File dir;
+
+ /** The mask. */
+ private final String filter;
+
+ /**
+ * Constructor.
+ *
+ * @param dir Directory.
+ * @param filter Filter.
+ */
+ private SearchDirectory(File dir, String filter) throws IOException {
+ this.dir = dir;
+ this.filter = filter;
+
+ if (!directoryExists(dir.getAbsolutePath()))
+ throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
+ }
+
+ /**
+ * @return Child files.
+ */
+ private File[] files() throws IOException {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return filter == null || name.startsWith(filter);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory. [dir=" + dir + ']');
+
+ return files;
+ }
+
+ /**
+ * @return {@code True} if filter exists.
+ */
+ private boolean hasFilter() {
+ return filter != null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/52841f90/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
new file mode 100644
index 0000000..b3e2d12
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Simple structure to hold Hadoop directory locations.
+ */
+public class HadoopLocations {
+ /** Hadoop home. */
+ private final String home;
+
+ /** Common home. */
+ private final String commonHome;
+
+ /** HDFS home. */
+ private final String hdfsHome;
+
+ /** Mapred home. */
+ private final String mapredHome;
+
+ /**
+ * Constructor.
+ *
+ * @param home Hadoop home.
+ * @param commonHome Common home.
+ * @param hdfsHome HDFS home.
+ * @param mapredHome Mapred home.
+ */
+ public HadoopLocations(String home, String commonHome, String hdfsHome, String mapredHome) {
+ this.home = home;
+ this.commonHome = commonHome;
+ this.hdfsHome = hdfsHome;
+ this.mapredHome = mapredHome;
+ }
+
+ /**
+ * @return Hadoop home.
+ */
+ public String home() {
+ return home;
+ }
+
+ /**
+ * @return Common home.
+ */
+ public String commonHome() {
+ return commonHome;
+ }
+
+ /**
+ * @return HDFS home.
+ */
+ public String hdfsHome() {
+ return hdfsHome;
+ }
+
+ /**
+ * @return Mapred home.
+ */
+ public String mapredHome() {
+ return mapredHome;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/52841f90/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c4cd143..c45953e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -33,7 +33,6 @@ import org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopEmbeddedT
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.internal.processors.hadoop.HadoopClasspathUtils.HadoopLocations;
/**
* Hadoop processor.
@@ -74,14 +73,14 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
try {
- HadoopLocations loc = HadoopClasspathUtils.getHadoopLocations();
+ HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
- if (loc.home != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home);
+ if (loc.home() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.common);
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfs);
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapred);
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
}
catch (IOException ioe) {
throw new IgniteCheckedException(ioe);
[25/27] ignite git commit: Merge branch 'gridgain-7.5.29' into
ignite-3185
Posted by vo...@apache.org.
Merge branch 'gridgain-7.5.29' into ignite-3185
# Conflicts:
# modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d7aa255a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d7aa255a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d7aa255a
Branch: refs/heads/ignite-3185
Commit: d7aa255a32bf336acb36f17d9d07bf5c30b2d79e
Parents: 057654a cc7ace9
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Jul 1 15:05:14 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Jul 1 15:05:14 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.bat | 4 +-
examples/config/filesystem/example-igfs.xml | 3 -
.../ignite/binary/BinaryObjectBuilder.java | 11 +
.../apache/ignite/internal/IgniteKernal.java | 53 +-
.../ignite/internal/binary/BinaryContext.java | 35 +
.../dht/GridDhtPartitionTopologyImpl.java | 2 +-
.../processors/hadoop/HadoopNoopProcessor.java | 5 +
.../hadoop/HadoopProcessorAdapter.java | 7 +
modules/core/src/test/config/igfs-loopback.xml | 7 -
modules/core/src/test/config/igfs-shmem.xml | 7 -
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 52 +-
.../query/h2/sql/GridQueryParsingTest.java | 11 +-
.../config/benchmark-cache-load-win.properties | 60 +
.../config/benchmark-cache-load.properties | 92 ++
.../yardstick/config/benchmark-full.properties | 116 ++
.../config/ignite-base-load-config.xml | 255 ++++
.../config/ignite-cache-load-config.xml | 71 ++
modules/yardstick/config/queries.sql | 2 +
.../yardstick/IgniteBenchmarkArguments.java | 50 +-
.../org/apache/ignite/yardstick/IgniteNode.java | 2 +
.../cache/IgniteCacheAbstractBenchmark.java | 59 +
.../yardstick/cache/IgniteGetTxBenchmark.java | 30 +
.../yardstick/cache/IgniteInvokeBenchmark.java | 65 +
.../cache/IgniteInvokeTxBenchmark.java | 30 +
.../IgniteCacheRandomOperationBenchmark.java | 1160 ++++++++++++++++++
.../yardstick/cache/load/model/ModelUtil.java | 181 +++
.../cache/load/model/key/Identifier.java | 113 ++
.../yardstick/cache/load/model/key/Mark.java | 115 ++
.../yardstick/cache/load/model/value/Car.java | 126 ++
.../yardstick/cache/load/model/value/Color.java | 50 +
.../yardstick/cache/load/model/value/Truck.java | 69 ++
.../ignite/yardstick/cache/model/Account.java | 6 +
.../ignite/yardstick/cache/model/Person1.java | 2 +-
.../ignite/yardstick/cache/model/Person2.java | 2 +-
.../ignite/yardstick/cache/model/SampleKey.java | 2 +-
.../yardstick/cache/model/SampleValue.java | 2 +-
parent/pom.xml | 1 +
38 files changed, 2797 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d7aa255a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --cc modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index bbf6e8e,7fc7499..8677c55
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@@ -204,6 -188,26 +189,25 @@@ public class HadoopProcessor extends Ha
return hctx.jobTracker().killJob(jobId);
}
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here.
+ try {
- HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
++ HadoopLocations loc = HadoopClasspathUtils.locations();
+
- if (loc.home() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
++ if (!F.isEmpty(loc.home()))
++ U.quietAndInfo(log, HadoopClasspathUtils.HOME + " is set to " + loc.home());
+
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
++ U.quietAndInfo(log, "Resolved Hadoop classpath locations: " + loc.common() + ", " + loc.hdfs() + ", " +
++ loc.mapred());
+ }
+ catch (IOException ioe) {
- throw new IgniteCheckedException(ioe.getMessage(), ioe);
++ throw new IgniteCheckedException(ioe);
+ }
+
+ HadoopClassLoader.hadoopUrls();
+ }
+
/**
* Initializes default hadoop configuration.
*
[24/27] ignite git commit: IGNITE-3410: IGFS: Added missing classes
to BInaryContext.
Posted by vo...@apache.org.
IGNITE-3410: IGFS: Added missing classes to BInaryContext.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cc7ace94
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cc7ace94
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cc7ace94
Branch: refs/heads/ignite-3185
Commit: cc7ace947a68b95598103249aedd36e5b5de1b5b
Parents: 1727647
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Jul 1 14:58:31 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Jul 1 14:58:31 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryContext.java | 35 ++++++++++++++++++++
1 file changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc7ace94/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index d39f9c6..593e8af 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -70,9 +70,25 @@ import org.apache.ignite.internal.processors.igfs.IgfsFileAffinityRange;
import org.apache.ignite.internal.processors.igfs.IgfsFileInfo;
import org.apache.ignite.internal.processors.igfs.IgfsFileMap;
import org.apache.ignite.internal.processors.igfs.IgfsListingEntry;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientAffinityCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientDeleteCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientExistsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientInfoCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientListFilesCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientListPathsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientMkdirsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientRenameCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSetTimesCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSizeCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSummaryCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientUpdateCallable;
+import org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaIdsForPathCallable;
+import org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaInfoForPathCallable;
+import org.apache.ignite.internal.processors.igfs.data.IgfsDataPutProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryListingAddProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryListingRemoveProcessor;
+import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryListingRenameProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryListingReplaceProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileLockProcessor;
@@ -126,9 +142,12 @@ public class BinaryContext {
sysClss.add(IgfsFileMap.class.getName());
sysClss.add(IgfsListingEntry.class.getName());
+ sysClss.add(IgfsDataPutProcessor.class.getName());
+
sysClss.add(IgfsMetaDirectoryCreateProcessor.class.getName());
sysClss.add(IgfsMetaDirectoryListingAddProcessor.class.getName());
sysClss.add(IgfsMetaDirectoryListingRemoveProcessor.class.getName());
+ sysClss.add(IgfsMetaDirectoryListingRenameProcessor.class.getName());
sysClss.add(IgfsMetaDirectoryListingReplaceProcessor.class.getName());
sysClss.add(IgfsMetaFileCreateProcessor.class.getName());
sysClss.add(IgfsMetaFileLockProcessor.class.getName());
@@ -139,6 +158,22 @@ public class BinaryContext {
sysClss.add(IgfsMetaUpdatePropertiesProcessor.class.getName());
sysClss.add(IgfsMetaUpdateTimesProcessor.class.getName());
+ sysClss.add(IgfsClientMetaIdsForPathCallable.class.getName());
+ sysClss.add(IgfsClientMetaInfoForPathCallable.class.getName());
+
+ sysClss.add(IgfsClientAffinityCallable.class.getName());
+ sysClss.add(IgfsClientDeleteCallable.class.getName());
+ sysClss.add(IgfsClientExistsCallable.class.getName());
+ sysClss.add(IgfsClientInfoCallable.class.getName());
+ sysClss.add(IgfsClientListFilesCallable.class.getName());
+ sysClss.add(IgfsClientListPathsCallable.class.getName());
+ sysClss.add(IgfsClientMkdirsCallable.class.getName());
+ sysClss.add(IgfsClientRenameCallable.class.getName());
+ sysClss.add(IgfsClientSetTimesCallable.class.getName());
+ sysClss.add(IgfsClientSizeCallable.class.getName());
+ sysClss.add(IgfsClientSummaryCallable.class.getName());
+ sysClss.add(IgfsClientUpdateCallable.class.getName());
+
// Closure processor classes.
sysClss.add(GridClosureProcessor.C1V2.class.getName());
sysClss.add(GridClosureProcessor.C1MLAV2.class.getName());
[27/27] ignite git commit: Merge branch 'ignite-3411' into ignite-3185
Posted by vo...@apache.org.
Merge branch 'ignite-3411' into ignite-3185
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/084abddb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/084abddb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/084abddb
Branch: refs/heads/ignite-3185
Commit: 084abddb2f3917e3a25b368048862e7fac2bc00e
Parents: c16c945 78bd095
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Fri Jul 1 15:08:10 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Jul 1 15:08:10 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopAttributes.java | 168 +++++++++++++++++++
.../processors/hadoop/HadoopProcessor.java | 49 +++---
2 files changed, 194 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/084abddb/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
[18/27] ignite git commit: merge to 7.5.28
Posted by vo...@apache.org.
merge to 7.5.28
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0bd836a8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0bd836a8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0bd836a8
Branch: refs/heads/ignite-3185
Commit: 0bd836a85b6fe78436fd962e14fdd0bc6a4b3bb9
Parents: 6ff658a 76c76f8
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Jun 27 18:51:16 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Jun 27 18:51:16 2016 +0300
----------------------------------------------------------------------
.../configuration/HadoopConfiguration.java | 6 +-
.../service/GridServiceProcessor.java | 2 +-
.../service/IgniteServiceReassignmentTest.java | 250 +++++++++++++++++++
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
.../IgniteHadoopClientProtocolProvider.java | 1 +
.../processors/hadoop/HadoopClasspathUtils.java | 13 +-
.../processors/hadoop/HadoopProcessor.java | 1 +
7 files changed, 268 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0bd836a8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0bd836a8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --cc modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index e84c23c,c45953e..ebd085a
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@@ -34,6 -34,6 +34,7 @@@ import org.apache.ignite.internal.util.
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
++import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
/**
* Hadoop processor.
*/
[07/27] ignite git commit: Merge branch 'gridgain-7.5.27' of
https://github.com/gridgain/apache-ignite into ignite-3184
Posted by vo...@apache.org.
Merge branch 'gridgain-7.5.27' of https://github.com/gridgain/apache-ignite into ignite-3184
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f7a7bc2f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f7a7bc2f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f7a7bc2f
Branch: refs/heads/ignite-3185
Commit: f7a7bc2f3c92f86cdcf408eac07b123ce13f23d2
Parents: 6910126 9881863
Author: iveselovskiy <iv...@gridgain.com>
Authored: Wed Jun 22 11:59:18 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Wed Jun 22 11:59:18 2016 +0300
----------------------------------------------------------------------
.../configuration/HadoopConfiguration.java | 35 ++++++++++++++++++++
.../processors/hadoop/HadoopJobInfo.java | 3 +-
.../processors/hadoop/HadoopClassLoader.java | 26 +++++++++++++--
.../processors/hadoop/HadoopDefaultJobInfo.java | 8 ++---
.../hadoop/jobtracker/HadoopJobTracker.java | 14 ++++++--
.../child/HadoopChildProcessRunner.java | 2 +-
.../processors/hadoop/v2/HadoopV2Job.java | 14 ++++++--
.../hadoop/HadoopClassLoaderTest.java | 2 +-
.../processors/hadoop/HadoopMapReduceTest.java | 3 ++
.../processors/hadoop/HadoopSnappyTest.java | 2 +-
.../processors/hadoop/HadoopTasksV1Test.java | 2 +-
.../processors/hadoop/HadoopTasksV2Test.java | 2 +-
.../processors/hadoop/HadoopV2JobSelfTest.java | 2 +-
.../collections/HadoopAbstractMapTest.java | 6 ++--
14 files changed, 99 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7a7bc2f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
[17/27] ignite git commit: 3185: corrected error messages.
Posted by vo...@apache.org.
3185: corrected error messages.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6ff658aa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6ff658aa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6ff658aa
Branch: refs/heads/ignite-3185
Commit: 6ff658aae7d2f4eb54f7dd6c4b71ea72f80c43d6
Parents: e575bf5
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Jun 27 16:58:54 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Jun 27 16:58:54 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 65 ++++++++++++++++----
.../processors/hadoop/HadoopLocations.java | 20 ++----
.../processors/hadoop/HadoopProcessor.java | 2 +-
3 files changed, 59 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6ff658aa/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 598c1c3..5eb5c47 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -83,7 +83,7 @@ public class HadoopClasspathUtils {
private static String hadoopHome() {
String prefix = systemOrEnv("HADOOP_PREFIX", null);
- return systemOrEnv("HADOOP_HOME", prefix);
+ return systemOrEnv(HadoopVariable.HADOOP_HOME.toString(), prefix);
}
/**
@@ -94,9 +94,9 @@ public class HadoopClasspathUtils {
private static HadoopLocations getEnvHadoopLocations() {
return new HadoopLocations(
hadoopHome(),
- systemOrEnv("HADOOP_COMMON_HOME", null),
- systemOrEnv("HADOOP_HDFS_HOME", null),
- systemOrEnv("HADOOP_MAPRED_HOME", null)
+ systemOrEnv(HadoopVariable.HADOOP_COMMON_HOME.toString(), null),
+ systemOrEnv(HadoopVariable.HADOOP_HDFS_HOME.toString(), null),
+ systemOrEnv(HadoopVariable.HADOOP_MAPRED_HOME.toString(), null)
);
}
@@ -116,8 +116,8 @@ public class HadoopClasspathUtils {
private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
"/usr/hdp/current/hadoop-client",
"/usr/hdp/current/hadoop-client",
- "/usr/hdp/current/hadoop-hdfs-client/",
- "/usr/hdp/current/hadoop-mapreduce-client/");
+ "/usr/hdp/current/hadoop-hdfs-client",
+ "/usr/hdp/current/hadoop-mapreduce-client");
/**
* HDP locations relative to an arbitrary Hadoop home.
@@ -127,8 +127,8 @@ public class HadoopClasspathUtils {
*/
private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
return new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs-client/",
- hadoopHome + "/../hadoop-mapreduce-client/");
+ hadoopHome + "/../hadoop-hdfs-client",
+ hadoopHome + "/../hadoop-mapreduce-client");
}
/**
@@ -136,7 +136,7 @@ public class HadoopClasspathUtils {
*
* @return The locations as determined from the environment.
*/
- private static HadoopLocations getHadoopLocations() throws IOException {
+ static HadoopLocations getHadoopLocations() throws IOException {
// 1. Get Hadoop locations from the environment:
HadoopLocations loc = getEnvHadoopLocations();
@@ -148,8 +148,7 @@ public class HadoopClasspathUtils {
if (hadoopHome != null) {
// If home is defined, it must exist:
if (!directoryExists(hadoopHome))
- throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
- + hadoopHome + ']');
+ throw ioeNotFound(HadoopVariable.HADOOP_HOME.toString(), hadoopHome);
// 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
loc = getApacheHadoopLocations(hadoopHome);
@@ -160,11 +159,34 @@ public class HadoopClasspathUtils {
// 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
loc = getHdpLocationsRelative(hadoopHome);
- return loc.existsOrException();
+ if (loc.exists())
+ return loc;
+
+ throw new IOException("Hadoop installation locations not found based on defined "
+ + HadoopVariable.HADOOP_HOME.toString() + " value. [" + HadoopVariable.HADOOP_HOME.toString()
+ + "=" + hadoopHome + ']');
}
// 4. Try absolute HDP (Hortonworks) location:
- return HDP_HADOOP_LOCATIONS.existsOrException();
+ if (HDP_HADOOP_LOCATIONS.exists())
+ return HDP_HADOOP_LOCATIONS;
+
+ throw new IOException("Hadoop installation locations cannot be found. Please define environment variables "
+ + HadoopVariable.HADOOP_COMMON_HOME.toString() + ", "
+ + HadoopVariable.HADOOP_HDFS_HOME.toString() + ", "
+ + HadoopVariable.HADOOP_MAPRED_HOME.toString() + ", or "
+ + HadoopVariable.HADOOP_HOME.toString() + ".");
+ }
+
+ /**
+ * Convenience method to construct Exception.
+ *
+ * @param var The variable name.
+ * @param dir The dir.
+ * @return The Exception.
+ */
+ static IOException ioeNotFound(String var, String dir) {
+ return new IOException(var + " location is not an existing readable directory. [" + var + "=" + dir + ']');
}
/**
@@ -271,4 +293,21 @@ public class HadoopClasspathUtils {
return filter != null;
}
}
+
+ /**
+ * Enumeration of commonly used Hadoop environment variables.
+ */
+ public enum HadoopVariable {
+ /** Variable HADOOP_HOME. */
+ HADOOP_HOME,
+
+ /** Variable HADOOP_COMMON_HOME. */
+ HADOOP_COMMON_HOME,
+
+ /** Variable HADOOP_HDFS_HOME. */
+ HADOOP_HDFS_HOME,
+
+ /** Variable HADOOP_MAPRED_HOME. */
+ HADOOP_MAPRED_HOME,
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6ff658aa/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
index 597a8dc..9be42a5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -105,25 +105,17 @@ public class HadoopLocations {
*/
public HadoopLocations existsOrException() throws IOException {
if (!HadoopClasspathUtils.directoryExists(commonHome))
- throw ioe("HADOOP_COMMON_HOME");
+ throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_COMMON_HOME.toString(),
+ commonHome);
if (!HadoopClasspathUtils.directoryExists(hdfsHome))
- throw ioe("HADOOP_HDFS_HOME");
+ throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_HDFS_HOME.toString(),
+ hdfsHome);
if (!HadoopClasspathUtils.directoryExists(mapredHome))
- throw ioe("HADOOP_MAPRED_HOME");
+ throw HadoopClasspathUtils.ioeNotFound(HadoopClasspathUtils.HadoopVariable.HADOOP_MAPRED_HOME.toString(),
+ mapredHome);
return this;
}
-
- /**
- * Constructs the exception.
- *
- * @param var The 1st variable name to mention in the exception text.
- * @return The exception.
- */
- private IOException ioe(String var) {
- return new IOException("Failed to resolve Hadoop installation location. " + var +
- " or HADOOP_HOME environment variable should be set.");
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6ff658aa/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c45953e..e84c23c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -73,7 +73,7 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
try {
- HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+ HadoopLocations loc = HadoopClasspathUtils.getHadoopLocations();
if (loc.home() != null)
U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
[09/27] ignite git commit: Merge remote-tracking branch
'upstream/ignite-3184' into ignite-3184
Posted by vo...@apache.org.
Merge remote-tracking branch 'upstream/ignite-3184' into ignite-3184
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7096957f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7096957f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7096957f
Branch: refs/heads/ignite-3185
Commit: 7096957fce8a6c3213ea5b5abbf5f284b9d25cdd
Parents: 50190d0 39c06ea
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 23 13:58:50 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 23 13:58:50 2016 +0300
----------------------------------------------------------------------
bin/include/hadoop-classpath.bat | 73 -
bin/include/hadoop-classpath.sh | 65 -
bin/include/setenv.bat | 6 +-
bin/include/setenv.sh | 30 +-
modules/clients/pom.xml | 8 -
.../client/ClientDefaultCacheSelfTest.java | 119 +-
.../ignite/internal/client/ClientHttpTask.java | 33 +-
.../ignite/internal/client/ClientTcpTask.java | 10 +-
.../integration/ClientAbstractSelfTest.java | 92 +-
.../JettyRestProcessorAbstractSelfTest.java | 1428 ++++++++++++------
.../internal/processors/rest/SimplePerson.java | 74 +
modules/core/pom.xml | 15 +-
.../configuration/HadoopConfiguration.java | 35 +
.../apache/ignite/internal/LessNamingBean.java | 28 +
.../cache/query/GridCacheSqlIndexMetadata.java | 3 +-
.../cache/query/GridCacheSqlMetadata.java | 3 +-
.../processors/hadoop/HadoopJobInfo.java | 3 +-
.../internal/util/IgniteExceptionRegistry.java | 5 +-
.../ignite/internal/util/IgniteUtils.java | 3 +-
.../ignite/internal/visor/cache/VisorCache.java | 10 +-
.../cache/VisorCacheAffinityConfiguration.java | 5 +-
.../cache/VisorCacheAggregatedMetrics.java | 3 +-
.../visor/cache/VisorCacheConfiguration.java | 3 +-
.../cache/VisorCacheDefaultConfiguration.java | 5 +-
.../cache/VisorCacheEvictionConfiguration.java | 5 +-
.../internal/visor/cache/VisorCacheMetrics.java | 3 +-
.../cache/VisorCacheNearConfiguration.java | 5 +-
.../visor/cache/VisorCachePartition.java | 3 +-
.../visor/cache/VisorCachePartitions.java | 3 +-
.../cache/VisorCacheQueryConfiguration.java | 3 +-
.../visor/cache/VisorCacheQueryMetrics.java | 5 +-
.../cache/VisorCacheRebalanceConfiguration.java | 5 +-
.../cache/VisorCacheStoreConfiguration.java | 3 +-
.../cache/VisorCacheTypeFieldMetadata.java | 3 +-
.../visor/cache/VisorCacheTypeMetadata.java | 3 +-
.../internal/visor/cache/VisorCacheV4.java | 124 ++
.../visor/compute/VisorGatewayTask.java | 361 +++++
.../internal/visor/debug/VisorThreadInfo.java | 5 +-
.../visor/debug/VisorThreadLockInfo.java | 5 +-
.../internal/visor/event/VisorGridEvent.java | 5 +-
.../internal/visor/file/VisorFileBlock.java | 5 +-
.../ignite/internal/visor/igfs/VisorIgfs.java | 5 +-
.../internal/visor/igfs/VisorIgfsEndpoint.java | 5 +-
.../internal/visor/igfs/VisorIgfsMetrics.java | 5 +-
.../visor/igfs/VisorIgfsProfilerEntry.java | 5 +-
.../VisorIgfsProfilerUniformityCounters.java | 5 +-
.../visor/log/VisorLogSearchResult.java | 5 +-
.../visor/node/VisorAtomicConfiguration.java | 5 +-
.../visor/node/VisorBasicConfiguration.java | 5 +-
.../node/VisorExecutorServiceConfiguration.java | 5 +-
.../visor/node/VisorGridConfiguration.java | 5 +-
.../visor/node/VisorIgfsConfiguration.java | 3 +-
.../visor/node/VisorLifecycleConfiguration.java | 5 +-
.../visor/node/VisorMetricsConfiguration.java | 5 +-
.../visor/node/VisorNodeDataCollectorJob.java | 28 +-
.../node/VisorNodeDataCollectorTaskResult.java | 5 +-
.../node/VisorPeerToPeerConfiguration.java | 5 +-
.../visor/node/VisorRestConfiguration.java | 5 +-
.../node/VisorSegmentationConfiguration.java | 5 +-
.../visor/node/VisorSpisConfiguration.java | 5 +-
.../node/VisorTransactionConfiguration.java | 5 +-
.../internal/visor/query/VisorQueryField.java | 5 +-
.../internal/visor/query/VisorQueryResult.java | 5 +-
.../internal/visor/util/VisorTaskUtils.java | 10 +-
.../plugin/security/SecurityPermissionSet.java | 5 +-
.../ignite/plugin/security/SecuritySubject.java | 5 +-
.../processors/hadoop/HadoopClassLoader.java | 47 +-
.../processors/hadoop/HadoopClasspathMain.java | 231 +--
.../processors/hadoop/HadoopClasspathUtils.java | 231 +++
.../processors/hadoop/HadoopDefaultJobInfo.java | 8 +-
.../processors/hadoop/HadoopProcessor.java | 9 +-
.../hadoop/jobtracker/HadoopJobTracker.java | 14 +-
.../child/HadoopChildProcessRunner.java | 2 +-
.../processors/hadoop/v2/HadoopV2Job.java | 14 +-
.../hadoop/HadoopClassLoaderTest.java | 2 +-
.../processors/hadoop/HadoopMapReduceTest.java | 3 +
.../processors/hadoop/HadoopSnappyTest.java | 2 +-
.../processors/hadoop/HadoopTasksV1Test.java | 2 +-
.../processors/hadoop/HadoopTasksV2Test.java | 2 +-
.../processors/hadoop/HadoopV2JobSelfTest.java | 2 +-
.../collections/HadoopAbstractMapTest.java | 6 +-
modules/rest-http/pom.xml | 25 +-
.../http/jetty/GridJettyJsonConfig.java | 195 ---
.../http/jetty/GridJettyObjectMapper.java | 274 ++++
.../http/jetty/GridJettyRestHandler.java | 127 +-
parent/pom.xml | 1 +
86 files changed, 2631 insertions(+), 1322 deletions(-)
----------------------------------------------------------------------
[15/27] ignite git commit: IGNITE-3185: implemented the logic.
Posted by vo...@apache.org.
IGNITE-3185: implemented the logic.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04f24348
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04f24348
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04f24348
Branch: refs/heads/ignite-3185
Commit: 04f24348bab2ad4f2a7ac45fbfae0d14236959d4
Parents: c401709 52841f9
Author: iveselovskiy <iv...@gridgain.com>
Authored: Fri Jun 24 19:20:11 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Fri Jun 24 19:20:11 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopClasspathMain.java | 16 +-
.../processors/hadoop/HadoopClasspathUtils.java | 288 +++++++++++--------
.../processors/hadoop/HadoopLocations.java | 78 +++++
.../processors/hadoop/HadoopProcessor.java | 13 +-
5 files changed, 259 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/04f24348/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --cc modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index acf19bd,a934f61..1a62a88
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@@ -56,237 -60,36 +60,206 @@@ public class HadoopClasspathUtils
* @return List of class path URLs.
* @throws IOException If failed.
*/
- public static List<URL> getAsUrlList() throws IOException {
- Collection<DirAndMask> dams = getClasspathBaseDirectories();
+ public static List<URL> classpathUrls() throws IOException {
+ List<URL> res = new ArrayList<>();
- List<URL> list = new ArrayList<>(32);
-
- for (DirAndMask dam: dams)
- // Note that this procedure does not use '*' classpath patterns,
- // but adds all the children explicitly:
- addUrls(list, dam.dir, dam.mask);
-
- return list;
- }
-
- /**
- * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
- *
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws IOException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory. [dir=" + dir + ']');
-
- for (File file : files) {
- try {
- res.add(file.toURI().toURL());
- }
- catch (MalformedURLException e) {
- throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ for (SearchDirectory dir : classpathDirectories()) {
+ for (File file : dir.files()) {
+ try {
+ res.add(file.toURI().toURL());
+ }
+ catch (MalformedURLException e) {
+ throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ }
}
}
- }
-
- /**
- * Discovers classpath entries in specified directory and adds them as URLs to the given {@code res} collection.
- *
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws IOException If failed.
- */
- private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith)
- throws IOException {
- if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
- throw new IOException("Path is not an existing readable directory. [dir=" + dir + ']');
- if (startsWith == null)
- res.add(dir.getAbsolutePath() + File.separator + '*');
- else {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory. [" + dir + ']');
-
- for (File file : files)
- res.add(file.getAbsolutePath());
- }
+ return res;
}
- /**
- * @return HADOOP_HOME Variable.
- */
- private static String hadoopHome() {
- String prefix = getEnv("HADOOP_PREFIX", null);
-
- return getEnv("HADOOP_HOME", prefix);
- }
++// /**
++// * @return HADOOP_HOME Variable.
++// */
++// private static String hadoopHome() {
++// String prefix = getEnv("HADOOP_PREFIX", null);
++//
++// return getEnv("HADOOP_HOME", prefix);
++// }
++
++// /**
++// * Simple structure to hold Hadoop directory locations.
++// */
++// public static class HadoopLocations {
++// /** HADOOP_HOME, may be null. */
++// public final String home;
++// /** HADOOP_COMMON_HOME */
++// public final String common;
++// /** HADOOP_HDFS_HOME */
++// public final String hdfs;
++// /** HADOOP_MAPRED_HOME */
++// public final String mapred;
++//
++// /**
++// * Constructor.
++// *
++// * @param home HADOOP_HOME
++// * @param common HADOOP_COMMON_HOME
++// * @param hdfs HADOOP_HDFS_HOME
++// * @param mapred HADOOP_MAPRED_HOME
++// */
++// HadoopLocations(String home, String common, String hdfs, String mapred) {
++// this.home = home;
++// this.common = common;
++// this.hdfs = hdfs;
++// this.mapred = mapred;
++// }
++// }
+
+ /**
+ * Simple structure to hold Hadoop directory locations.
+ */
+ public static class HadoopLocations {
+ /** HADOOP_HOME, may be null. */
+ public final String home;
+ /** HADOOP_COMMON_HOME */
+ public final String common;
+ /** HADOOP_HDFS_HOME */
+ public final String hdfs;
+ /** HADOOP_MAPRED_HOME */
+ public final String mapred;
+
+ /**
+ * Constructor.
+ *
+ * @param home HADOOP_HOME
+ * @param common HADOOP_COMMON_HOME
+ * @param hdfs HADOOP_HDFS_HOME
+ * @param mapred HADOOP_MAPRED_HOME
+ */
+ HadoopLocations(String home, String common, String hdfs, String mapred) {
+ this.home = home;
+ this.common = common;
+ this.hdfs = hdfs;
+ this.mapred = mapred;
+ }
+
+ /**
+ * Answers if all the base directories are defined.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+ */
+ public boolean isDefined() {
+ return common != null && hdfs != null && mapred != null;
+ }
+
+ /**
+ * Answers if all the base directories exist.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+ */
+ public boolean exists() {
+ return isExistingDirectory(common)
+ && isExistingDirectory(hdfs)
+ && isExistingDirectory(mapred);
+ }
+
+ /**
+ * Checks if all the base directories exist.
+ *
+ * @return this reference.
+ * @throws IOException if any of the base directories does not exist.
+ */
+ public HadoopLocations existsOrException() throws IOException {
+ if (!isExistingDirectory(common))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_COMMON_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(hdfs))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_HDFS_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(mapred))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_MAPRED_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ return this;
+ }
+ }
+
+ /**
+ * Gets locations from the environment.
+ *
+ * @return The locations as determined from the environment.
+ */
+ private static HadoopLocations getEnvHadoopLocations() {
+ return new HadoopLocations(
+ hadoopHome(),
+ getEnv("HADOOP_COMMON_HOME", null),
+ getEnv("HADOOP_HDFS_HOME", null),
+ getEnv("HADOOP_MAPRED_HOME", null)
+ );
+ }
+
+ /**
+ * Gets locations assuming Apache Hadoop distribution layout.
+ *
+ * @return The locations as for Apache distribution.
+ */
+ private static HadoopLocations getApacheHadoopLocations(String hadoopHome) {
+ return new HadoopLocations(hadoopHome,
+ hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs",
+ hadoopHome + "/share/hadoop/mapreduce");
+ }
+
+ /** HDP Hadoop locations. */
+ private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-hdfs-client/",
+ "/usr/hdp/current/hadoop-mapreduce-client/");
+
+ /**
+ * HDP locations relative to an arbitrary Hadoop home.
+ *
+ * @param hadoopHome The hadoop home.
+ * @return The locations.
+ */
+ private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
+ return new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client/",
+ hadoopHome + "/../hadoop-mapreduce-client/");
+ }
-
++
++
/**
- * Gets the existing Hadoop locations, if any.
+ * Gets Hadoop locations.
*
- * @return Existing Hadoop locations.
- * @throws IOException If no existing location found.
- * @return The Hadoop locations, never null.
++ * @return The locations as determined from the environment.
*/
- public static HadoopLocations hadoopLocations() throws IOException {
- final String hadoopHome = systemOrEnv("HADOOP_HOME", systemOrEnv("HADOOP_PREFIX", null));
+ public static HadoopLocations getHadoopLocations() throws IOException {
- // 1. Try locations defined in System properties or environment:
- HadoopLocations loc = getEnvHadoopLocations();
-
- if (loc.isDefined())
- return loc.existsOrException();
-
+ final String hadoopHome = hadoopHome();
+
+ if (hadoopHome != null) {
+ // If home is defined, it must exist:
+ if (!isExistingDirectory(hadoopHome))
+ throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
+ + hadoopHome + ']');
+
+ // 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
+ loc = getApacheHadoopLocations(hadoopHome);
- String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
- String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
- String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+ if (loc.exists())
+ return loc;
- return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+ // 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
+ loc = getHdpLocationsRelative(hadoopHome);
+
+ return loc.existsOrException();
+ }
+
+ // 4. Try absolute HDP (Hortonworks) location:
+ return HDP_HADOOP_LOCATIONS.existsOrException();
}
/**
@@@ -295,46 -98,51 +268,73 @@@
* @return Collection of directory and mask pairs.
* @throws IOException if a mandatory classpath location is not found.
*/
- private static Collection<DirAndMask> getClasspathBaseDirectories() throws IOException {
- HadoopLocations loc = getHadoopLocations();
+ private static Collection<SearchDirectory> classpathDirectories() throws IOException {
+ HadoopLocations loc = hadoopLocations();
- Collection<DirAndMask> c = new ArrayList<>();
+ Collection<SearchDirectory> res = new ArrayList<>();
- c.add(new DirAndMask(new File(loc.common, "lib"), null));
- c.add(new DirAndMask(new File(loc.hdfs, "lib"), null));
- c.add(new DirAndMask(new File(loc.mapred, "lib"), null));
+ res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null));
- c.add(new DirAndMask(new File(loc.common), "hadoop-common-"));
- c.add(new DirAndMask(new File(loc.common), "hadoop-auth-"));
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-"));
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-"));
- c.add(new DirAndMask(new File(loc.hdfs), "hadoop-hdfs-"));
+ res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-"));
- c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-common"));
- c.add(new DirAndMask(new File(loc.mapred), "hadoop-mapreduce-client-core"));
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common"));
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core"));
- return c;
+ return res;
}
++// /**
++// * Simple pair-like structure to hold directory name and a mask assigned to it.
++// */
++// public static class DirAndMask {
++// /**
++// * Constructor.
++// *
++// * @param dir The directory.
++// * @param mask The mask.
++// */
++// DirAndMask(File dir, String mask) {
++// this.dir = dir;
++// this.mask = mask;
++// }
++//
++// /** The path. */
++// public final File dir;
++//
++// /** The mask. */
++// public final String mask;
++// }
++
/**
- * Simple pair-like structure to hold directory name and a mask assigned to it.
+ * Resolves a Hadoop location directory.
+ *
+ * @param envVarName Environment variable name. The value denotes the location path.
+ * @param hadoopHome Hadoop home location, may be null.
+ * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
+ * @throws IOException If the value cannot be resolved to an existing directory.
*/
- public static class DirAndMask {
- /**
- * Constructor.
- *
- * @param dir The directory.
- * @param mask The mask.
- */
- DirAndMask(File dir, String mask) {
- this.dir = dir;
- this.mask = mask;
+ private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
+ throws IOException {
- String val = systemOrEnv(envVarName, null);
++ String val = getEnv(envVarName, null);
+
+ if (val == null) {
+ // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!directoryExists(hadoopHome))
++ if (!isExistingDirectory(hadoopHome))
+ throw new IOException("Failed to resolve Hadoop installation location. " +
+ envVarName + " or HADOOP_HOME environment variable should be set.");
+
+ val = hadoopHome + expHadoopHomeRelativePath;
}
- /** The path. */
- public final File dir;
- if (!directoryExists(val))
++ if (!isExistingDirectory(val))
+ throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
- /** The mask. */
- public final String mask;
+ return val;
}
/**
[22/27] ignite git commit: Minors.
Posted by vo...@apache.org.
Minors.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04ea922f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04ea922f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04ea922f
Branch: refs/heads/ignite-3185
Commit: 04ea922f949308b5ca3a559d0ab0512d05216c50
Parents: 75601f8
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 13:17:57 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 13:17:57 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/04ea922f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index c83231e..f84b093 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -135,21 +135,21 @@ public class HadoopClasspathUtils {
hadoopHome + "/share/hadoop/hdfs", hadoopHome + "/share/hadoop/mapreduce"));
// Probe for CDH with normal HADOOP_HOME directory.
+ probes.add(new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs", hadoopHome + "/../hadoop-mapreduce"));
+
+ // Probe for CDH with HADOOP_HOME set to parent directory.
probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop",
hadoopHome + "/hadoop-hdfs", hadoopHome + "/hadoop-mapreduce"));
- // Probe for CDH with HADOOP_HOME equal to HADOOP_COMMON_HOME.
+ // Probe for HDP with normal HADOOP_HOME directory.
probes.add(new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs", hadoopHome + "/../hadoop-mapreduce"));
+ hadoopHome + "/../hadoop-hdfs-client", hadoopHome + "/../hadoop-mapreduce-client"));
- // Probe for HDP with normal HADOOP_HOME directory.
+ // Probe for HDP with HADOOP_HOME set to parent directory.
probes.add(new HadoopLocations(hadoopHome, hadoopHome + "/hadoop-client,",
hadoopHome + "/hadoop-hdfs-client", hadoopHome + "/hadoop-mapreduce-client"));
- // Probe for HDP with HADOOP_HOME equal to HADOOP_COMMON_HOME.
- probes.add(new HadoopLocations(hadoopHome, hadoopHome,
- hadoopHome + "/../hadoop-hdfs-client", hadoopHome + "/../hadoop-mapreduce-client"));
-
for (HadoopLocations probe : probes) {
if (probe.valid())
return probe;
[04/27] ignite git commit: IGNITE-3184: minor corrections.
Posted by vo...@apache.org.
IGNITE-3184: minor corrections.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/aa5f5a6d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/aa5f5a6d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/aa5f5a6d
Branch: refs/heads/ignite-3185
Commit: aa5f5a6d99d000eccd7d8840c901963a559bd7cf
Parents: f0d08f8
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Jun 20 19:41:55 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Jun 20 19:41:55 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.bat | 2 +-
.../main/java/org/apache/ignite/internal/util/IgniteUtils.java | 3 +--
.../ignite/internal/processors/hadoop/HadoopClassLoader.java | 4 ++--
.../ignite/internal/processors/hadoop/HadoopClasspathUtils.java | 1 -
4 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/aa5f5a6d/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 7022a36..dec1ee5 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -23,7 +23,7 @@
:: in other scripts to set classpath using exported IGNITE_LIBS variable.
::
-:: @echo off
+@echo off
:: USER_LIBS variable can optionally contain user's JARs/libs.
:: set USER_LIBS=
http://git-wip-us.apache.org/repos/asf/ignite/blob/aa5f5a6d/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index c28e7bc..f802d94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4049,8 +4049,7 @@ public abstract class IgniteUtils {
public static void quietAndWarn(IgniteLogger log, Object msg, Object shortMsg) {
warn(log, msg);
- if (log != null
- && log.isQuiet())
+ if (log != null && log.isQuiet())
quiet(false, shortMsg);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/aa5f5a6d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 8b6ea34..62f9668 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -168,9 +168,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
ldr = ldr.getParent();
}
}
- catch (Throwable t) {
+ catch (Exception e) {
U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
- "(native Hadoop methods might not work properly): " + t);
+ "(native Hadoop methods might not work properly): " + e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/aa5f5a6d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index c4a2fcc..9f6d6b1 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -137,7 +137,6 @@ public class HadoopClasspathUtils {
c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
- c.add(new DirAndMask(new File(commonHome, "lib"), "hadoop-auth-"));
c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
[02/27] ignite git commit: Merge branch 'gridgain-7.5.27' into
ignite-3184
Posted by vo...@apache.org.
Merge branch 'gridgain-7.5.27' into ignite-3184
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/50190d05
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/50190d05
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/50190d05
Branch: refs/heads/ignite-3185
Commit: 50190d058660d622deaa23690f8a3667f139120e
Parents: 0c7b08c c952def
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 20 12:24:22 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 20 12:24:22 2016 +0300
----------------------------------------------------------------------
.../processors/igfs/IgfsDeleteWorker.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 99 ++------------
.../igfs/client/IgfsClientAbstractCallable.java | 17 ---
.../igfs/client/IgfsClientDeleteCallable.java | 7 -
.../igfs/client/IgfsClientMkdirsCallable.java | 6 -
.../igfs/client/IgfsClientRenameCallable.java | 6 -
.../igfs/client/IgfsClientSetTimesCallable.java | 7 -
.../igfs/client/IgfsClientUpdateCallable.java | 6 -
.../meta/IgfsClientMetaFileUnlockCallable.java | 130 -------------------
9 files changed, 13 insertions(+), 267 deletions(-)
----------------------------------------------------------------------
[10/27] ignite git commit: Merge branch 'gridgain-7.5.27' into
ignite-3184
Posted by vo...@apache.org.
Merge branch 'gridgain-7.5.27' into ignite-3184
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7bbaafcf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7bbaafcf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7bbaafcf
Branch: refs/heads/ignite-3185
Commit: 7bbaafcf916bf0dac6fba8c489003c4ca4fecfa0
Parents: 7096957 60a2e28
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 23 13:59:43 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 23 13:59:43 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/IgnitionEx.java | 3 +
.../processors/cache/GridCacheAdapter.java | 16 +-
.../distributed/near/GridNearGetFuture.java | 8 +-
.../transactions/IgniteTxLocalAdapter.java | 12 +-
.../cache/transactions/IgniteTxManager.java | 16 +-
.../processors/igfs/IgfsDataManager.java | 14 +-
.../processors/igfs/IgfsOutputStreamImpl.java | 10 +-
.../internal/processors/igfs/IgfsProcessor.java | 24 --
.../ignite/internal/util/IgniteUtils.java | 45 +++-
.../ignite/spi/discovery/tcp/ServerImpl.java | 7 +-
.../spi/discovery/tcp/TcpDiscoveryImpl.java | 3 +-
.../IgniteCacheTxIteratorSelfTest.java | 241 +++++++++++++++++++
.../cluster/GridAddressResolverSelfTest.java | 97 ++++++++
.../igfs/IgfsDataManagerSelfTest.java | 6 +-
.../igfs/IgfsProcessorValidationSelfTest.java | 26 --
.../vm/TcpDiscoveryVmIpFinderSelfTest.java | 75 ++++++
.../testsuites/IgniteCacheTestSuite5.java | 2 +
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
18 files changed, 501 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7bbaafcf/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------