You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/07/01 12:10:57 UTC
[14/27] ignite git commit: IGNITE-3185
IGNITE-3185
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c4017097
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c4017097
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c4017097
Branch: refs/heads/ignite-3185
Commit: c4017097bd443069090626acba4772de01051f98
Parents: 9cc1607
Author: iveselovskiy <iv...@gridgain.com>
Authored: Fri Jun 24 17:26:32 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Fri Jun 24 17:26:32 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClasspathUtils.java | 148 ++++++++++++++-----
1 file changed, 115 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c4017097/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index bb7ccd3..acf19bd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -163,21 +163,130 @@ public class HadoopClasspathUtils {
this.hdfs = hdfs;
this.mapred = mapred;
}
+
+ /**
+ * Answers if all the base directories are defined.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+ */
+ public boolean isDefined() {
+ return common != null && hdfs != null && mapred != null;
+ }
+
+ /**
+ * Answers if all the base directories exist.
+ *
+ * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+ */
+ public boolean exists() {
+ return isExistingDirectory(common)
+ && isExistingDirectory(hdfs)
+ && isExistingDirectory(mapred);
+ }
+
+ /**
+ * Checks if all the base directories exist.
+ *
+ * @return this reference.
+ * @throws IOException if any of the base directories does not exist.
+ */
+ public HadoopLocations existsOrException() throws IOException {
+ if (!isExistingDirectory(common))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_COMMON_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(hdfs))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_HDFS_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ if (!isExistingDirectory(mapred))
+ throw new IOException("Failed to resolve Hadoop installation location. HADOOP_MAPRED_HOME " +
+ "or HADOOP_HOME environment variable should be set.");
+
+ return this;
+ }
+ }
+
+ /**
+ * Gets locations from the environment.
+ *
+ * @return The locations as determined from the environment.
+ */
+ private static HadoopLocations getEnvHadoopLocations() {
+ return new HadoopLocations(
+ hadoopHome(),
+ getEnv("HADOOP_COMMON_HOME", null),
+ getEnv("HADOOP_HDFS_HOME", null),
+ getEnv("HADOOP_MAPRED_HOME", null)
+ );
+ }
+
+ /**
+ * Gets locations assuming Apache Hadoop distribution layout.
+ *
+ * @return The locations as for Apache distribution.
+ */
+ private static HadoopLocations getApacheHadoopLocations(String hadoopHome) {
+ return new HadoopLocations(hadoopHome,
+ hadoopHome + "/share/hadoop/common",
+ hadoopHome + "/share/hadoop/hdfs",
+ hadoopHome + "/share/hadoop/mapreduce");
+ }
+
+ /** HDP Hadoop locations. */
+ private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-client",
+ "/usr/hdp/current/hadoop-hdfs-client/",
+ "/usr/hdp/current/hadoop-mapreduce-client/");
+
+ /**
+ * HDP locations relative to an arbitrary Hadoop home.
+ *
+ * @param hadoopHome The hadoop home.
+ * @return The locations.
+ */
+ private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
+ return new HadoopLocations(hadoopHome, hadoopHome,
+ hadoopHome + "/../hadoop-hdfs-client/",
+ hadoopHome + "/../hadoop-mapreduce-client/");
}
/**
- * Gets Hadoop locations.
+ * Gets the existing Hadoop locations, if any.
*
- * @return The Hadoop locations, never null.
+ * @return Existing Hadoop locations.
+ * @throws IOException If no existing location found.
*/
public static HadoopLocations getHadoopLocations() throws IOException {
+ // 1. Try locations defined in System properties or environment:
+ HadoopLocations loc = getEnvHadoopLocations();
+
+ if (loc.isDefined())
+ return loc.existsOrException();
+
final String hadoopHome = hadoopHome();
- String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
- String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
- String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+ if (hadoopHome != null) {
+ // If home is defined, it must exist:
+ if (!isExistingDirectory(hadoopHome))
+ throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
+ + hadoopHome + ']');
+
+ // 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
+ loc = getApacheHadoopLocations(hadoopHome);
- return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+ if (loc.exists())
+ return loc;
+
+ // 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
+ loc = getHdpLocationsRelative(hadoopHome);
+
+ return loc.existsOrException();
+ }
+
+ // 4. Try absolute HDP (Hortonworks) location:
+ return HDP_HADOOP_LOCATIONS.existsOrException();
}
/**
@@ -229,33 +338,6 @@ public class HadoopClasspathUtils {
}
/**
- * Resolves a Hadoop location directory.
- *
- * @param envVarName Environment variable name. The value denotes the location path.
- * @param hadoopHome Hadoop home location, may be null.
- * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
- * @throws IOException If the value cannot be resolved to an existing directory.
- */
- private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
- throws IOException {
- String val = getEnv(envVarName, null);
-
- if (val == null) {
- // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
- if (!isExistingDirectory(hadoopHome))
- throw new IOException("Failed to resolve Hadoop installation location. " +
- envVarName + " or HADOOP_HOME environment variable should be set.");
-
- val = hadoopHome + expHadoopHomeRelativePath;
- }
-
- if (!isExistingDirectory(val))
- throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
-
- return val;
- }
-
- /**
* Note that this method does not treat empty value as an absent value.
*
* @param name Variable name.