You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/06/28 12:12:27 UTC
[6/6] ignite git commit: IGNITE-3383: Implemented.
IGNITE-3383: Implemented.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5317ef3a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5317ef3a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5317ef3a
Branch: refs/heads/ignite-3383
Commit: 5317ef3ad0db55c7e32290dde6215d99d5dea436
Parents: a77e690
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 15:12:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 15:12:13 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 43 +++++++++++-----
.../processors/hadoop/HadoopNoopProcessor.java | 5 ++
.../hadoop/HadoopProcessorAdapter.java | 7 +++
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 52 +++++++++-----------
5 files changed, 66 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5317ef3a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3572293..9098fca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -115,6 +115,7 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.hadoop.Hadoop;
import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor;
@@ -1087,26 +1088,44 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* Create Hadoop component.
*
* @return Non-null Hadoop component: workable or no-op.
- * @throws IgniteCheckedException if the component is mandatory and cannot be initialized.
+ * @throws IgniteCheckedException If the component is mandatory and cannot be initialized.
*/
- private GridComponent createHadoopComponent() throws IgniteCheckedException {
- GridComponent cmp;
+ private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedException {
+ boolean mandatory = cfg.getHadoopConfiguration() != null;
- if (cfg.isPeerClassLoadingEnabled()) {
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+ if (mandatory) {
+ if (cfg.isPeerClassLoadingEnabled())
+ throw new IgniteCheckedException("Hadoop module cannot be user with peer class loading enabled " +
+ "(set set IgniteConfiguration.peerClassLoadingEnabled to \"false\").");
- if (!(cmp instanceof HadoopNoopProcessor)) {
- U.warn(log, "Hadoop module is found in classpath, but it will not be started because peer class " +
+ return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+ }
+ else {
+ HadoopProcessorAdapter cmp = null;
+
+ if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled()) {
+ U.warn(log, "Hadoop module is found in classpath, but will not be started because peer class " +
"loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" to start " +
"Hadoop module).");
+ }
+ else {
+ cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+
+ try {
+ cmp.validateEnvironment();
+ }
+ catch (IgniteException | IgniteCheckedException e) {
+ U.quietAndWarn(log, "Hadoop module will not start due to exception: " + e.getMessage());
- cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/);
+ cmp = null;
+ }
}
- }
- else
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration() != null);
- return cmp;
+ if (cmp == null)
+ cmp = IgniteComponentType.HADOOP.create(ctx, true);
+
+ return cmp;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5317ef3a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
index 72b5050..065cd72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -74,4 +74,9 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter {
@Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
return false;
}
+
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // No-oop.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5317ef3a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
index 3d381c3..f914639 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
@@ -94,4 +94,11 @@ public abstract class HadoopProcessorAdapter extends GridProcessorAdapter {
* @throws IgniteCheckedException If failed.
*/
public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Validate Hadoop environment.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void validateEnvironment() throws IgniteCheckedException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5317ef3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 1c844c4..389de8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
hadoopUrls = HadoopClasspathUtils.classpathUrls();
}
catch (IOException e) {
- throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e);
+ throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations: " + e.getMessage(), e);
}
hadoopJars = hadoopUrls;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5317ef3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c45953e..7fc7499 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -50,6 +50,8 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
private Hadoop hadoop;
/**
+ * Constructor.
+ *
* @param ctx Kernal context.
*/
public HadoopProcessor(GridKernalContext ctx) {
@@ -70,24 +72,6 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
initializeDefaults(cfg);
- validate(cfg);
-
- try {
- HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
-
- if (loc.home() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
-
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
- }
- catch (IOException ioe) {
- throw new IgniteCheckedException(ioe);
- }
-
- HadoopClassLoader.hadoopUrls();
-
hctx = new HadoopContext(
ctx,
cfg,
@@ -204,6 +188,26 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
return hctx.jobTracker().killJob(jobId);
}
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here.
+ try {
+ HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+
+ if (loc.home() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
+
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+ }
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe.getMessage(), ioe);
+ }
+
+ HadoopClassLoader.hadoopUrls();
+ }
+
/**
* Initializes default hadoop configuration.
*
@@ -213,16 +217,4 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
if (cfg.getMapReducePlanner() == null)
cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner());
}
-
- /**
- * Validates Grid and Hadoop configuration for correctness.
- *
- * @param hadoopCfg Hadoop configuration.
- * @throws IgniteCheckedException If failed.
- */
- private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException {
- if (ctx.config().isPeerClassLoadingEnabled())
- throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop (disable it using " +
- "IgniteConfiguration.setPeerClassLoadingEnabled()).");
- }
}
\ No newline at end of file