You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/22 10:46:13 UTC
ignite git commit: IGNITE-3954: Hadoop: fixing tests.
Repository: ignite
Updated Branches:
refs/heads/ignite-1.6.8-hadoop 4c09b38cc -> 778878a03
IGNITE-3954: Hadoop: fixing tests.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/778878a0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/778878a0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/778878a0
Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 778878a03df987a65071c92c2990fe7f4c30231d
Parents: 4c09b38
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Sep 22 13:46:04 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Sep 22 13:46:04 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 8 +-
.../processors/hadoop/HadoopSharedMap.java | 67 +++++++++++
.../hadoop/HadoopTestClassLoader.java | 89 ++++++++++++++
.../hadoop/impl/HadoopGroupingTest.java | 36 +++---
.../hadoop/impl/HadoopJobTrackerSelfTest.java | 22 +---
.../impl/HadoopMapReduceEmbeddedSelfTest.java | 7 +-
.../processors/hadoop/impl/HadoopSharedMap.java | 66 -----------
.../impl/HadoopTaskExecutionSelfTest.java | 32 +-----
.../hadoop/state/HadoopGroupingTestState.java | 40 +++++++
.../state/HadoopJobTrackerSelfTestState.java | 45 ++++++++
.../HadoopMapReduceEmbeddedSelfTestState.java | 32 ++++++
.../HadoopTaskExecutionSelfTestValues.java | 51 ++++++++
.../testsuites/IgniteHadoopTestSuite.java | 3 +-
.../testsuites/IgniteHadoopTestSuite2.java | 115 -------------------
.../IgniteIgfsLinuxAndMacOSTestSuite.java | 3 +-
15 files changed, 355 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index d2c9014..60f5f55 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -347,7 +347,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
* @return Whether class must be loaded by current classloader without delegation.
*/
@SuppressWarnings("RedundantIfStatement")
- private static boolean loadByCurrentClassloader(String clsName) {
+ public static boolean loadByCurrentClassloader(String clsName) {
// All impl classes.
if (clsName.startsWith("org.apache.ignite.internal.processors.hadoop.impl"))
return true;
@@ -361,12 +361,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
return true;
}
- // TODO: Move suites to "impl" package.
- // Test suites (to be removed).
- if (clsName.equals("org.apache.ignite.testsuites.IgniteHadoopTestSuite") ||
- clsName.equals("org.apache.ignite.testsuites.IgniteIgfsLinuxAndMacOSTestSuite"))
- return true;
-
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
new file mode 100644
index 0000000..fd72821
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.util.concurrent.ConcurrentMap;
+import org.jsr166.ConcurrentHashMap8;
+
+/**
+ * For tests.
+ */
+public class HadoopSharedMap {
+ /** */
+ private static final ConcurrentMap<String, HadoopSharedMap> maps = new ConcurrentHashMap8<>();
+
+ /** */
+ private final ConcurrentMap<String, Object> map = new ConcurrentHashMap8<>();
+
+ /**
+ * Private.
+ */
+ private HadoopSharedMap() {
+ // No-op.
+ }
+
+ /**
+ * Puts object by key.
+ *
+ * @param key Key.
+ * @param val Value.
+ */
+ @SuppressWarnings("unchecked")
+ public <T> T put(String key, T val) {
+ Object old = map.putIfAbsent(key, val);
+
+ return old == null ? val : (T)old;
+ }
+
+ /**
+ * @param cls Class.
+ * @return Map of static fields.
+ */
+ public static HadoopSharedMap map(Class<?> cls) {
+ HadoopSharedMap m = maps.get(cls.getName());
+
+ if (m != null)
+ return m;
+
+ HadoopSharedMap old = maps.putIfAbsent(cls.getName(), m = new HadoopSharedMap());
+
+ return old == null ? m : old;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestClassLoader.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestClassLoader.java
new file mode 100644
index 0000000..35bb9b0
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestClassLoader.java
@@ -0,0 +1,89 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Hadoop test class loader aimed to provide better isolation.
+ */
+public class HadoopTestClassLoader extends URLClassLoader {
+ /** Parent class loader. */
+ private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)HadoopTestClassLoader.class.getClassLoader();
+
+ /** */
+ private static final Collection<URL> APP_JARS = F.asList(APP_CLS_LDR.getURLs());
+
+ /** All participating URLs. */
+ private static final URL[] URLS;
+
+ static {
+ try {
+ List<URL> res = new ArrayList<>();
+
+ for (URL url : APP_JARS) {
+ String urlStr = url.toString();
+
+ if (urlStr.contains("modules/hadoop-impl/"))
+ res.add(url);
+ }
+
+ res.addAll(HadoopClasspathUtils.classpathForClassLoader());
+
+ X.println(">>> " + HadoopTestClassLoader.class.getSimpleName() + " static paths:");
+
+ for (URL url : res)
+ X.println(">>> \t" + url.toString());
+
+ URLS = res.toArray(new URL[res.size()]);
+ }
+ catch (Exception e) {
+ throw new IgniteException("Failed to initialize class loader JARs.", e);
+ }
+ }
+
+ /**
+ * Constructor.
+ */
+ public HadoopTestClassLoader() {
+ super(URLS, APP_CLS_LDR);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ if (HadoopClassLoader.loadByCurrentClassloader(name)) {
+ try {
+ synchronized (getClassLoadingLock(name)) {
+ // First, check if the class has already been loaded
+ Class c = findLoadedClass(name);
+
+ if (c == null) {
+ long t1 = System.nanoTime();
+
+ c = findClass(name);
+
+ // this is the defining class loader; record the stats
+ sun.misc.PerfCounter.getFindClassTime().addElapsedTimeFrom(t1);
+ sun.misc.PerfCounter.getFindClasses().increment();
+ }
+
+ if (resolve)
+ resolveClass(c);
+
+ return c;
+ }
+ }
+ catch (NoClassDefFoundError | ClassNotFoundException e) {
+ throw new IgniteException("Failed to load class by test class loader: " + name, e);
+ }
+ }
+
+ return super.loadClass(name, resolve);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopGroupingTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopGroupingTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopGroupingTest.java
index d2e3418..19c71e8 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopGroupingTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopGroupingTest.java
@@ -17,15 +17,6 @@
package org.apache.ignite.internal.processors.hadoop.impl;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
@@ -41,25 +32,28 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.ignite.configuration.HadoopConfiguration;
+import org.apache.ignite.internal.processors.hadoop.state.HadoopGroupingTestState;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.S;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
/**
* Grouping test.
*/
public class HadoopGroupingTest extends HadoopAbstractSelfTest {
- /** */
- private static final String PATH_OUTPUT = "/test-out";
-
- /** */
- private static final GridConcurrentHashSet<UUID> vals = HadoopSharedMap.map(HadoopGroupingTest.class)
- .put("vals", new GridConcurrentHashSet<UUID>());
-
/** {@inheritDoc} */
@Override protected int gridCount() {
return 3;
@@ -109,7 +103,7 @@ public class HadoopGroupingTest extends HadoopAbstractSelfTest {
* @throws Exception If failed.
*/
public void doTestGrouping(boolean combiner) throws Exception {
- vals.clear();
+ HadoopGroupingTestState.values().clear();
Job job = Job.getInstance();
@@ -135,7 +129,7 @@ public class HadoopGroupingTest extends HadoopAbstractSelfTest {
grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2),
createJobInfo(job.getConfiguration())).get(30000);
- assertTrue(vals.isEmpty());
+ assertTrue(HadoopGroupingTestState.values().isEmpty());
}
public static class MyReducer extends Reducer<YearTemperature, Text, Text, Object> {
@@ -161,7 +155,7 @@ public class HadoopGroupingTest extends HadoopAbstractSelfTest {
lastYear = key.year;
for (Text val : vals0)
- assertTrue(vals.remove(UUID.fromString(val.toString())));
+ assertTrue(HadoopGroupingTestState.values().remove(UUID.fromString(val.toString())));
}
}
@@ -268,7 +262,7 @@ public class HadoopGroupingTest extends HadoopAbstractSelfTest {
@Override public Text getCurrentValue() {
UUID id = UUID.randomUUID();
- assertTrue(vals.add(id));
+ assertTrue(HadoopGroupingTestState.values().add(id));
val.set(id.toString());
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
index fbb54ed..a3bf49c 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
@@ -21,12 +21,9 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -47,6 +44,10 @@ import org.apache.ignite.internal.processors.hadoop.HadoopJobStatus;
import org.apache.ignite.internal.util.typedef.internal.U;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopJobTrackerSelfTestState.combineExecCnt;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopJobTrackerSelfTestState.latch;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopJobTrackerSelfTestState.mapExecCnt;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopJobTrackerSelfTestState.reduceExecCnt;
/**
* Job tracker self test.
@@ -58,21 +59,6 @@ public class HadoopJobTrackerSelfTest extends HadoopAbstractSelfTest {
/** Test block count parameter name. */
private static final int BLOCK_CNT = 10;
- /** */
- private static HadoopSharedMap m = HadoopSharedMap.map(HadoopJobTrackerSelfTest.class);
-
- /** Map task execution count. */
- private static final AtomicInteger mapExecCnt = m.put("mapExecCnt", new AtomicInteger());
-
- /** Reduce task execution count. */
- private static final AtomicInteger reduceExecCnt = m.put("reduceExecCnt", new AtomicInteger());
-
- /** Reduce task execution count. */
- private static final AtomicInteger combineExecCnt = m.put("combineExecCnt", new AtomicInteger());
-
- /** */
- private static final Map<String, CountDownLatch> latch = m.put("latch", new HashMap<String, CountDownLatch>());
-
/** {@inheritDoc} */
@Override protected boolean igfsEnabled() {
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
index 1ce45cd..b04deeb 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
@@ -17,8 +17,6 @@
package org.apache.ignite.internal.processors.hadoop.impl;
-import java.util.HashMap;
-import java.util.Map;
import java.util.UUID;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -40,15 +38,12 @@ import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCoun
import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount2;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopMapReduceEmbeddedSelfTestState.flags;
/**
* Tests map-reduce execution with embedded mode.
*/
public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
- /** */
- private static Map<String, Boolean> flags = HadoopSharedMap.map(HadoopMapReduceEmbeddedSelfTest.class)
- .put("flags", new HashMap<String, Boolean>());
-
/** {@inheritDoc} */
@Override public HadoopConfiguration hadoopConfiguration(String gridName) {
HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopSharedMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopSharedMap.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopSharedMap.java
deleted file mode 100644
index 26f6fae..0000000
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopSharedMap.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.impl;
-
-import java.util.concurrent.ConcurrentMap;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- * For tests.
- */
-public class HadoopSharedMap {
- /** */
- private static final ConcurrentMap<String, HadoopSharedMap> maps = new ConcurrentHashMap8<>();
-
- /** */
- private final ConcurrentMap<String, Object> map = new ConcurrentHashMap8<>();
-
- /**
- * Private.
- */
- private HadoopSharedMap() {
- // No-op.
- }
-
- /**
- * Puts object by key.
- *
- * @param key Key.
- * @param val Value.
- */
- public <T> T put(String key, T val) {
- Object old = map.putIfAbsent(key, val);
-
- return old == null ? val : (T)old;
- }
-
- /**
- * @param cls Class.
- * @return Map of static fields.
- */
- public static HadoopSharedMap map(Class<?> cls) {
- HadoopSharedMap m = maps.get(cls.getName());
-
- if (m != null)
- return m;
-
- HadoopSharedMap old = maps.putIfAbsent(cls.getName(), m = new HadoopSharedMap());
-
- return old == null ? m : old;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
index d2c75b4..027f921 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
@@ -22,11 +22,8 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -56,38 +53,21 @@ import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.cancelledTasks;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.executedTasks;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.failMapperId;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.splitsCount;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.taskWorkDirs;
+import static org.apache.ignite.internal.processors.hadoop.state.HadoopTaskExecutionSelfTestValues.totalLineCnt;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
/**
* Tests map-reduce task execution basics.
*/
public class HadoopTaskExecutionSelfTest extends HadoopAbstractSelfTest {
- /** */
- private static HadoopSharedMap m = HadoopSharedMap.map(HadoopTaskExecutionSelfTest.class);
-
- /** Line count. */
- private static final AtomicInteger totalLineCnt = m.put("totalLineCnt", new AtomicInteger());
-
- /** Executed tasks. */
- private static final AtomicInteger executedTasks = m.put("executedTasks", new AtomicInteger());
-
- /** Cancelled tasks. */
- private static final AtomicInteger cancelledTasks = m.put("cancelledTasks", new AtomicInteger());
-
- /** Working directory of each task. */
- private static final Map<String, String> taskWorkDirs = m.put("taskWorkDirs",
- new ConcurrentHashMap<String, String>());
-
- /** Mapper id to fail. */
- private static final AtomicInteger failMapperId = m.put("failMapperId", new AtomicInteger());
-
- /** Number of splits of the current input. */
- private static final AtomicInteger splitsCount = m.put("splitsCount", new AtomicInteger());
-
/** Test param. */
private static final String MAP_WRITE = "test.map.write";
-
/** {@inheritDoc} */
@Override public FileSystemConfiguration igfsConfiguration() throws Exception {
FileSystemConfiguration cfg = super.igfsConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
new file mode 100644
index 0000000..af82d49
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Shared state for HadoopGroupingTest.
+ */
+public class HadoopGroupingTestState {
+ /** Values. */
+ private static final GridConcurrentHashSet<UUID> vals = HadoopSharedMap.map(HadoopGroupingTestState.class)
+ .put("vals", new GridConcurrentHashSet<UUID>());
+
+ /**
+ * @return Values.
+ */
+ public static Collection<UUID> values() {
+ return vals;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
new file mode 100644
index 0000000..4183f4e
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Shared state for HadoopJobTrackerSelfTest.
+ */
+public class HadoopJobTrackerSelfTestState {
+ /** */
+ private static HadoopSharedMap m = HadoopSharedMap.map(HadoopJobTrackerSelfTestState.class);
+
+ /** Map task execution count. */
+ public static final AtomicInteger mapExecCnt = m.put("mapExecCnt", new AtomicInteger());
+
+ /** Reduce task execution count. */
+ public static final AtomicInteger reduceExecCnt = m.put("reduceExecCnt", new AtomicInteger());
+
+ /** Reduce task execution count. */
+ public static final AtomicInteger combineExecCnt = m.put("combineExecCnt", new AtomicInteger());
+
+ /** */
+ public static final Map<String, CountDownLatch> latch = m.put("latch", new HashMap<String, CountDownLatch>());
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
new file mode 100644
index 0000000..ae6ce82
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Sharead state for HadoopMapReduceEmbeddedSelfTest.
+ */
+public class HadoopMapReduceEmbeddedSelfTestState {
+ /** */
+ public static Map<String, Boolean> flags = HadoopSharedMap.map(HadoopMapReduceEmbeddedSelfTestState.class)
+ .put("flags", new HashMap<String, Boolean>());
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
new file mode 100644
index 0000000..beefa68
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Shared state for HadoopTaskExecutionSelfTest.
+ */
+public class HadoopTaskExecutionSelfTestValues {
+ /** */
+ private static HadoopSharedMap m = HadoopSharedMap.map(HadoopTaskExecutionSelfTestValues.class);
+
+ /** Line count. */
+ public static final AtomicInteger totalLineCnt = m.put("totalLineCnt", new AtomicInteger());
+
+ /** Executed tasks. */
+ public static final AtomicInteger executedTasks = m.put("executedTasks", new AtomicInteger());
+
+ /** Cancelled tasks. */
+ public static final AtomicInteger cancelledTasks = m.put("cancelledTasks", new AtomicInteger());
+
+ /** Working directory of each task. */
+ public static final Map<String, String> taskWorkDirs = m.put("taskWorkDirs",
+ new ConcurrentHashMap<String, String>());
+
+ /** Mapper id to fail. */
+ public static final AtomicInteger failMapperId = m.put("failMapperId", new AtomicInteger());
+
+ /** Number of splits of the current input. */
+ public static final AtomicInteger splitsCount = m.put("splitsCount", new AtomicInteger());
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 944e285..bbd92d1 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -22,6 +22,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolEmbeddedSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTxConfigCacheTest;
@@ -106,7 +107,7 @@ public class IgniteHadoopTestSuite extends TestSuite {
downloadHadoop();
downloadHive();
- final ClassLoader ldr = TestSuite.class.getClassLoader();
+ final ClassLoader ldr = new HadoopTestClassLoader();
TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite2.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite2.java
deleted file mode 100644
index 1df5765..0000000
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite2.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package org.apache.ignite.testsuites;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
-import org.apache.ignite.internal.util.typedef.F;
-
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Separate class loader for better class handling.
- */
-// TODO: Remove or adopt.
-public class IgniteHadoopTestSuite2 extends TestSuite {
- /**
- * @return Test suite.
- * @throws Exception Thrown in case of the failure.
- */
- public static TestSuite suite() throws Exception {
- final ClassLoader ldr = new TestClassLoader();
-
- TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
-
- suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolSelfTest.class.getName())));
-
- return suite;
- }
-
- /**
- * Class loader for tests.
- */
- private static class TestClassLoader extends URLClassLoader {
- /** Parent class loader. */
- private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)TestClassLoader.class.getClassLoader();
-
- /** */
- private static final Collection<URL> APP_JARS = F.asList(APP_CLS_LDR.getURLs());
-
- /** All participating URLs. */
- private static final URL[] URLS;
-
- static {
- try {
- List<URL> allJars = new ArrayList<>();
-
- allJars.addAll(APP_JARS);
-
- // TODO: Remove
- //allJars.addAll(HadoopClasspathUtils.classpathForClassLoader());
-
- List<URL> res = new ArrayList<>();
-
- for (URL url : allJars) {
- String urlStr = url.toString();
-
- if (urlStr.contains("modules/hadoop-impl/") ||
- urlStr.contains("org/apache/hadoop")) {
- res.add(url);
-
- System.out.println(url.toString());
- }
- }
-
- URLS = res.toArray(new URL[res.size()]);
- }
- catch (Exception e) {
- throw new IgniteException("Failed to initialize classloader JARs.", e);
- }
- }
-
- /**
- * Constructor.
- *
- * @throws Exception If failed.
- */
- public TestClassLoader() throws Exception {
- super(URLS, APP_CLS_LDR);
- }
-
- /** {@inheritDoc} */
- @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
- try {
- synchronized (getClassLoadingLock(name)) {
- // First, check if the class has already been loaded
- Class c = findLoadedClass(name);
-
- if (c == null) {
- long t1 = System.nanoTime();
-
- c = findClass(name);
-
- // this is the defining class loader; record the stats
- sun.misc.PerfCounter.getFindClassTime().addElapsedTimeFrom(t1);
- sun.misc.PerfCounter.getFindClasses().increment();
- }
-
- if (resolve)
- resolveClass(c);
-
- return c;
- }
- }
- catch (NoClassDefFoundError | ClassNotFoundException e) {
- // TODO: Remove
- //System.out.println("Not founded, delegated: " + name);
- }
-
- return super.loadClass(name, resolve);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/778878a0/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
index 0d4bf3e..21d78ab 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfs20FileSystemShmemPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgfsEventsTestSuite;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemIpcCacheSelfTest;
@@ -45,7 +46,7 @@ public class IgniteIgfsLinuxAndMacOSTestSuite extends TestSuite {
public static TestSuite suite() throws Exception {
downloadHadoop();
- ClassLoader ldr = TestSuite.class.getClassLoader();
+ final ClassLoader ldr = new HadoopTestClassLoader();
TestSuite suite = new TestSuite("Ignite IGFS Test Suite For Linux And Mac OS");