You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/06/02 12:38:10 UTC
[40/41] incubator-ignite git commit: #[IGNITE-218]: cleanup.
#[IGNITE-218]: cleanup.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/fa239a8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/fa239a8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/fa239a8f
Branch: refs/heads/ignite-218
Commit: fa239a8f6364889d735820ffc7408d21696ce46e
Parents: bf32347
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Jun 2 13:36:51 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Jun 2 13:36:51 2015 +0300
----------------------------------------------------------------------
config/hadoop/default-config.xml | 3 --
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 7 +---
.../internal/processors/hadoop/HadoopUtils.java | 44 ++------------------
.../processors/hadoop/v2/HadoopV2Job.java | 2 +-
.../hadoop/v2/HadoopV2JobResourceManager.java | 2 -
.../hadoop/HadoopAbstractSelfTest.java | 7 +---
.../hadoop/HadoopCommandLineTest.java | 14 ++++++-
.../processors/hadoop/HadoopMapReduceTest.java | 32 ++++++--------
8 files changed, 33 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/config/hadoop/default-config.xml
----------------------------------------------------------------------
diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml
index 54073fc..30413b0 100644
--- a/config/hadoop/default-config.xml
+++ b/config/hadoop/default-config.xml
@@ -90,9 +90,6 @@
Configuration of Ignite node.
-->
<bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
- <!-- TODO: Temporary workaround for tests: -->
- <property name="localHost" value="127.0.0.1"/>
-
<!--
Apache Hadoop Accelerator configuration.
-->
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index ec4a26f..4f703d2 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -358,12 +358,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
FileSystem cached = get(getUri(), getConf());
if (cached == this)
- return;
- else {
- X.println("### Cache enabled, but this file system is not found in the cache: " +
- " this = " + this + ", user =" + this.user +
- " cached = " + cached + ", user =" + ((IgniteHadoopFileSystem)cached).user);
- }
+ return; // do not close cached instances.
}
close0();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index 12015af..9085051 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -303,42 +303,10 @@ public class HadoopUtils {
return new File(jobLocDir, info.type() + "_" + info.taskNumber() + "_" + info.attempt());
}
- // TODO: after disagniostic & testing leave only one method "safeCreateConfiguration()"
/**
- *
- * @return
- */
- static Configuration checkPreconditionAndCreateConfiguration() {
- ClassLoader confCl = Configuration.class.getClassLoader();
-
- ClassLoader ctxCl = Thread.currentThread().getContextClassLoader();
-
- if (ctxCl != null && confCl != ctxCl)
- throw new IllegalStateException("Wrong classloader: " + confCl + " != " + ctxCl);
-
- return new Configuration();
- }
-
- /**
- * For diagnostic & test purposes.
- * @param c the Configuration to check.
- */
- static void checkConfiguration(Configuration c) {
- String name = Configuration.class.getName();
-
- c.set("xxx", name);
-
- Class clazz = c.getClass("xxx", null);
-
- c.unset("xxx");
-
- if (clazz != c.getClass())
- throw new IllegalStateException("Wrong configuration.");
- }
-
- /**
- *
- * @return
+ * Creates {@link Configuration} in a correct class loader context to avoid caching
+ * of inappropriate class loader in the Configuration object.
+ * @return New instance of {@link Configuration}.
*/
public static Configuration safeCreateConfiguration() {
final ClassLoader cl0 = Thread.currentThread().getContextClassLoader();
@@ -346,11 +314,7 @@ public class HadoopUtils {
Thread.currentThread().setContextClassLoader(Configuration.class.getClassLoader());
try {
- Configuration c = checkPreconditionAndCreateConfiguration();
-
- checkConfiguration(c);
-
- return c;
+ return new Configuration();
}
finally {
Thread.currentThread().setContextClassLoader(cl0);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index 3d47960..34ba053 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -331,7 +331,7 @@ public class HadoopV2Job implements HadoopJob {
/**
* Getter for job configuration.
- * @return the job configuration
+ * @return The job configuration.
*/
public JobConf jobConf() {
return jobConf;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
index 8f1e1ab..75602dc 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
@@ -105,8 +105,6 @@ public class HadoopV2JobResourceManager {
if (user == null)
user = IgniteHadoopFileSystem.getFsHadoopUser();
- X.println("##### Mr user = [" + user + "]"); // TODO: remove
-
return user;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
index 7aa2bde..e8a0a6f 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
@@ -61,15 +61,10 @@ public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
/** Initial REST port. */
private int restPort = REST_PORT;
- /** Secondary file system REST endpoint configuration map. */
+ /** Secondary file system REST endpoint configuration. */
protected static final IgfsIpcEndpointConfiguration SECONDARY_REST_CFG;
static {
-// PRIMARY_REST_CFG = new IgfsIpcEndpointConfiguration();
-//
-// PRIMARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
-// PRIMARY_REST_CFG.setPort(10500);
-
SECONDARY_REST_CFG = new IgfsIpcEndpointConfiguration();
SECONDARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
index d10ee5c..c66cdf3 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
@@ -19,12 +19,16 @@ package org.apache.ignite.internal.processors.hadoop;
import com.google.common.base.*;
import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
import org.apache.ignite.hadoop.fs.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.processors.resource.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
import org.apache.ignite.testframework.junits.common.*;
import org.jsr166.*;
@@ -205,7 +209,15 @@ public class HadoopCommandLineTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
- igfs = (IgfsEx) Ignition.start("config/hadoop/default-config.xml").fileSystem(igfsName);
+ String cfgPath = "config/hadoop/default-config.xml";
+
+ IgniteBiTuple<IgniteConfiguration, GridSpringResourceContext> tup = IgnitionEx.loadConfiguration(cfgPath);
+
+ IgniteConfiguration cfg = tup.get1();
+
+ cfg.setLocalHost("127.0.0.1"); // Avoid connecting to other nodes.
+
+ igfs = (IgfsEx) Ignition.start(cfg).fileSystem(igfsName);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fa239a8f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
index b0a098d..a1ef7ba 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
@@ -69,14 +69,16 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
/** Secondary file system configuration path. */
protected static final String SECONDARY_CFG = "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml";
+ /** The user to run Hadoop job on behalf of. */
protected static final String USER = "vasya";
+ /** Secondary IGFS name. */
protected static final String SECONDARY_IGFS_NAME = "igfs-secondary";
- /** */
+ /** The secondary Ignite node. */
protected Ignite igniteSecondary;
- /** */
+ /** The secondary Fs. */
protected IgfsSecondaryFileSystem secondaryFs;
/** {@inheritDoc} */
@@ -85,19 +87,19 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
}
/**
- *
- * @param p
- * @return
+ * Gets owner of a IgfsEx path.
+ * @param p The path.
+ * @return The owner.
*/
private static String getOwner(IgfsEx i, IgfsPath p) {
return i.info(p).property(IgfsEx.PROP_USER_NAME);
}
/**
- *
- * @param secFs
- * @param p
- * @return
+ * Gets owner of a secondary Fs path.
+ * @param secFs The sec Fs.
+ * @param p The path.
+ * @return The owner.
*/
private static String getOwnerSecondary(final IgfsSecondaryFileSystem secFs, final IgfsPath p) {
return IgfsUserContext.doAs(USER, new IgniteOutClosure<String>() {
@@ -108,8 +110,8 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
}
/**
- *
- * @param p
+ * Checks owner of the path.
+ * @param p The path.
*/
private void checkOwner(IgfsPath p) {
String ownerPrim = getOwner(igfs, p);
@@ -141,9 +143,6 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
JobConf jobConf = new JobConf();
- //jobConf.setBoolean("fs.igfs.impl.disable.cache", true); // avoid hangup in shutdown hook
- // when fs.close() causes the fs to be created again. // This does not work, why?????
-
jobConf.set(JOB_COUNTER_WRITER_PROPERTY, IgniteHadoopFileSystemCounterWriter.class.getName());
jobConf.setUser(USER);
jobConf.set(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY, "/xxx/${USER}/zzz");
@@ -373,9 +372,4 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
return fsCfg;
}
-
-// /** {@inheritDoc} */
-// @Override protected long getTestTimeout() {
-// return 30 * 60 * 1000; // TODO: for testing
-// }
}