You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/09/21 14:52:55 UTC
[03/92] [abbrv] [partial] ignite git commit: Moving classes around.
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolEmbeddedSelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolEmbeddedSelfTest.java
new file mode 100644
index 0000000..a65d691
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolEmbeddedSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.client;
+
+import org.apache.ignite.configuration.HadoopConfiguration;
+
+/**
+ * Hadoop client protocol tests in embedded process mode.
+ */
+public class HadoopClientProtocolEmbeddedSelfTest extends HadoopClientProtocolSelfTest {
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ // TODO: IGNITE-404: Uncomment when fixed.
+ //cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
new file mode 100644
index 0000000..b039d5e
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.client;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.StringTokenizer;
+import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
+import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider;
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopAbstractSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Hadoop client protocol tests in external process mode.
+ */
+@SuppressWarnings("ResultOfMethodCallIgnored")
+public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
+ /** Input path. */
+ private static final String PATH_INPUT = "/input";
+
+ /** Output path. */
+ private static final String PATH_OUTPUT = "/output";
+
+ /** Job name. */
+ private static final String JOB_NAME = "myJob";
+
+ /** Setup lock file. */
+ private static File setupLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-setup.file");
+
+ /** Map lock file. */
+ private static File mapLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-map.file");
+
+ /** Reduce lock file. */
+ private static File reduceLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-reduce.file");
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean restEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(gridCount());
+
+ setupLockFile.delete();
+ mapLockFile.delete();
+ reduceLockFile.delete();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+
+// IgniteHadoopClientProtocolProvider.cliMap.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ setupLockFile.createNewFile();
+ mapLockFile.createNewFile();
+ reduceLockFile.createNewFile();
+
+ setupLockFile.deleteOnExit();
+ mapLockFile.deleteOnExit();
+ reduceLockFile.deleteOnExit();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ grid(0).fileSystem(HadoopAbstractSelfTest.igfsName).format();
+
+ setupLockFile.delete();
+ mapLockFile.delete();
+ reduceLockFile.delete();
+
+ super.afterTest();
+ }
+
+ /**
+ * Test next job ID generation.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void tstNextJobId() throws Exception {
+ IgniteHadoopClientProtocolProvider provider = provider();
+
+ ClientProtocol proto = provider.create(config(HadoopAbstractSelfTest.REST_PORT));
+
+ JobID jobId = proto.getNewJobID();
+
+ assert jobId != null;
+ assert jobId.getJtIdentifier() != null;
+
+ JobID nextJobId = proto.getNewJobID();
+
+ assert nextJobId != null;
+ assert nextJobId.getJtIdentifier() != null;
+
+ assert !F.eq(jobId, nextJobId);
+ }
+
+ /**
+ * Tests job counters retrieval.
+ *
+ * @throws Exception If failed.
+ */
+ public void testJobCounters() throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+
+ igfs.mkdirs(new IgfsPath(PATH_INPUT));
+
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
+ new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
+
+ bw.write(
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n" +
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n" +
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n"
+ );
+ }
+
+ Configuration conf = config(HadoopAbstractSelfTest.REST_PORT);
+
+ final Job job = Job.getInstance(conf);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestCountingMapper.class);
+ job.setReducerClass(TestCountingReducer.class);
+ job.setCombinerClass(TestCountingCombiner.class);
+
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+
+ job.submit();
+
+ final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
+
+ assertEquals(0, cntr.getValue());
+
+ cntr.increment(10);
+
+ assertEquals(10, cntr.getValue());
+
+ // Transferring to map phase.
+ setupLockFile.delete();
+
+ // Transferring to reduce phase.
+ mapLockFile.delete();
+
+ job.waitForCompletion(false);
+
+ assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
+
+ final Counters counters = job.getCounters();
+
+ assertNotNull("counters cannot be null", counters);
+ assertEquals("wrong counters count", 3, counters.countCounters());
+ assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+ }
+
+ /**
+ * Tests job counters retrieval for unknown job id.
+ *
+ * @throws Exception If failed.
+ */
+ private void tstUnknownJobCounters() throws Exception {
+ IgniteHadoopClientProtocolProvider provider = provider();
+
+ ClientProtocol proto = provider.create(config(HadoopAbstractSelfTest.REST_PORT));
+
+ try {
+ proto.getJobCounters(new JobID(UUID.randomUUID().toString(), -1));
+ fail("exception must be thrown");
+ }
+ catch (Exception e) {
+ assert e instanceof IOException : "wrong error has been thrown";
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMap() throws Exception {
+ checkJobSubmit(true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapCombine() throws Exception {
+ checkJobSubmit(false, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapReduce() throws Exception {
+ checkJobSubmit(true, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapCombineReduce() throws Exception {
+ checkJobSubmit(false, false);
+ }
+
+ /**
+ * Test job submission.
+ *
+ * @param noCombiners Whether there are no combiners.
+ * @param noReducers Whether there are no reducers.
+ * @throws Exception If failed.
+ */
+ public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+
+ igfs.mkdirs(new IgfsPath(PATH_INPUT));
+
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
+ new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
+
+ bw.write("word");
+ }
+
+ Configuration conf = config(HadoopAbstractSelfTest.REST_PORT);
+
+ final Job job = Job.getInstance(conf);
+
+ job.setJobName(JOB_NAME);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
+
+ if (!noCombiners)
+ job.setCombinerClass(TestCombiner.class);
+
+ if (noReducers)
+ job.setNumReduceTasks(0);
+
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(TestOutputFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+
+ job.submit();
+
+ JobID jobId = job.getJobID();
+
+ // Setup phase.
+ JobStatus jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
+ assert jobStatus.getMapProgress() == 0.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
+
+ U.sleep(2100);
+
+ JobStatus recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() :
+ "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
+
+ // Transferring to map phase.
+ setupLockFile.delete();
+
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getSetupProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
+ }
+ }, 5000L);
+
+ // Map phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
+
+ U.sleep(2100);
+
+ recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() :
+ "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
+
+ // Transferring to reduce phase.
+ mapLockFile.delete();
+
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getMapProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
+ }
+ }, 5000L);
+
+ if (!noReducers) {
+ // Reduce phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
+
+ // Ensure that reduces progress increases.
+ U.sleep(2100);
+
+ recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() :
+ "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
+
+ reduceLockFile.delete();
+ }
+
+ job.waitForCompletion(false);
+
+ jobStatus = job.getStatus();
+ checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() == 1.0f;
+
+ dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+ }
+
+ /**
+ * Dump IGFS content.
+ *
+ * @param igfs IGFS.
+ * @param path Path.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private static void dumpIgfs(IgniteFileSystem igfs, IgfsPath path) throws Exception {
+ IgfsFile file = igfs.info(path);
+
+ assert file != null;
+
+ System.out.println(file.path());
+
+ if (file.isDirectory()) {
+ for (IgfsPath child : igfs.listPaths(path))
+ dumpIgfs(igfs, child);
+ }
+ else {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(igfs.open(path)))) {
+ String line = br.readLine();
+
+ while (line != null) {
+ System.out.println(line);
+
+ line = br.readLine();
+ }
+ }
+ }
+ }
+
+ /**
+ * Check job status.
+ *
+ * @param status Job status.
+ * @param expJobId Expected job ID.
+ * @param expJobName Expected job name.
+ * @param expState Expected state.
+ * @param expCleanupProgress Expected cleanup progress.
+ * @throws Exception If failed.
+ */
+ private static void checkJobStatus(JobStatus status, JobID expJobId, String expJobName,
+ JobStatus.State expState, float expCleanupProgress) throws Exception {
+ assert F.eq(status.getJobID(), expJobId) : "Expected=" + expJobId + ", actual=" + status.getJobID();
+ assert F.eq(status.getJobName(), expJobName) : "Expected=" + expJobName + ", actual=" + status.getJobName();
+ assert F.eq(status.getState(), expState) : "Expected=" + expState + ", actual=" + status.getState();
+ assert F.eq(status.getCleanupProgress(), expCleanupProgress) :
+ "Expected=" + expCleanupProgress + ", actual=" + status.getCleanupProgress();
+ }
+
+ /**
+ * @return Configuration.
+ */
+ private Configuration config(int port) {
+ Configuration conf = HadoopUtils.safeCreateConfiguration();
+
+ setupFileSystems(conf);
+
+ conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);
+ conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);
+
+ conf.set("fs.defaultFS", "igfs://:" + getTestGridName(0) + "@/");
+
+ return conf;
+ }
+
+ /**
+ * @return Protocol provider.
+ */
+ private IgniteHadoopClientProtocolProvider provider() {
+ return new IgniteHadoopClientProtocolProvider();
+ }
+
+ /**
+ * Test mapper.
+ */
+ public static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ while (mapLockFile.exists())
+ Thread.sleep(50);
+
+ StringTokenizer wordList = new StringTokenizer(val.toString());
+
+ while (wordList.hasMoreTokens()) {
+ word.set(wordList.nextToken());
+
+ ctx.write(word, one);
+ }
+ }
+ }
+
+ /**
+ * Test Hadoop counters.
+ */
+ public enum TestCounter {
+ COUNTER1, COUNTER2, COUNTER3
+ }
+
+ /**
+ * Test mapper that uses counters.
+ */
+ public static class TestCountingMapper extends TestMapper {
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ super.map(key, val, ctx);
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ }
+ }
+
+ /**
+ * Test combiner that counts invocations.
+ */
+ public static class TestCountingCombiner extends TestReducer {
+ @Override public void reduce(Text key, Iterable<IntWritable> values,
+ Context ctx) throws IOException, InterruptedException {
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ ctx.getCounter(TestCounter.COUNTER2).increment(1);
+
+ int sum = 0;
+ for (IntWritable value : values)
+ sum += value.get();
+
+ ctx.write(key, new IntWritable(sum));
+ }
+ }
+
+ /**
+ * Test reducer that counts invocations.
+ */
+ public static class TestCountingReducer extends TestReducer {
+ @Override public void reduce(Text key, Iterable<IntWritable> values,
+ Context ctx) throws IOException, InterruptedException {
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ ctx.getCounter(TestCounter.COUNTER3).increment(1);
+ }
+ }
+
+ /**
+ * Test combiner.
+ */
+ public static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
+ // No-op.
+ }
+
+ public static class TestOutputFormat<K, V> extends TextOutputFormat<K, V> {
+ /** {@inheritDoc} */
+ @Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx)
+ throws IOException {
+ return new TestOutputCommitter(ctx, (FileOutputCommitter)super.getOutputCommitter(ctx));
+ }
+ }
+
+ /**
+ * Test output committer.
+ */
+ private static class TestOutputCommitter extends FileOutputCommitter {
+ /** Delegate. */
+ private final FileOutputCommitter delegate;
+
+ /**
+ * Constructor.
+ *
+ * @param ctx Task attempt context.
+ * @param delegate Delegate.
+ * @throws IOException If failed.
+ */
+ private TestOutputCommitter(TaskAttemptContext ctx, FileOutputCommitter delegate) throws IOException {
+ super(FileOutputFormat.getOutputPath(ctx), ctx);
+
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setupJob(JobContext jobCtx) throws IOException {
+ try {
+ while (setupLockFile.exists())
+ Thread.sleep(50);
+ }
+ catch (InterruptedException ignored) {
+ throw new IOException("Interrupted.");
+ }
+
+ delegate.setupJob(jobCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setupTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.setupTask(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean needsTaskCommit(TaskAttemptContext taskCtx) throws IOException {
+ return delegate.needsTaskCommit(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void commitTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.commitTask(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void abortTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.abortTask(taskCtx);
+ }
+ }
+
+ /**
+ * Test reducer.
+ */
+ public static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ /** Writable container for writing sum of word counts. */
+ private IntWritable totalWordCnt = new IntWritable();
+
+ /** {@inheritDoc} */
+ @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
+ InterruptedException {
+ while (reduceLockFile.exists())
+ Thread.sleep(50);
+
+ int wordCnt = 0;
+
+ for (IntWritable value : values)
+ wordCnt += value.get();
+
+ totalWordCnt.set(wordCnt);
+
+ ctx.write(key, totalWordCnt);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWIthHadoop.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWIthHadoop.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWIthHadoop.java
new file mode 100644
index 0000000..25aef42
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWIthHadoop.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.mapreduce.Job;
+
+/**
+ * Class has a direct Hadoop dependency and a circular dependency on another class.
+ */
+@SuppressWarnings("unused")
+public class CircularWIthHadoop {
+ /** */
+ private Job[][] jobs = new Job[4][4];
+
+ /** */
+ private CircularWithoutHadoop y;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWithoutHadoop.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWithoutHadoop.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWithoutHadoop.java
new file mode 100644
index 0000000..a20cda9
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/CircularWithoutHadoop.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+/**
+ * Does not have direct Hadoop dependency, but has a circular
+ */
+@SuppressWarnings("unused")
+public class CircularWithoutHadoop {
+ /** */
+ private CircularWIthHadoop x;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithCast.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithCast.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithCast.java
new file mode 100644
index 0000000..cf800b5
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithCast.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Class contains casting to a Hadoop type.
+ */
+@SuppressWarnings("unused")
+public abstract class WithCast<T> {
+ /** */
+ public abstract T create();
+
+ /** */
+ public void consume(T t) {
+ // noop
+ }
+
+ /** */
+ void test(WithCast<FileSystem> c) {
+ FileSystem fs = c.create();
+
+ c.consume(fs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithClassAnnotation.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithClassAnnotation.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithClassAnnotation.java
new file mode 100644
index 0000000..b005448
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithClassAnnotation.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Class has Hadoop annotation.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Public
+public class WithClassAnnotation {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithConstructorInvocation.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithConstructorInvocation.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithConstructorInvocation.java
new file mode 100644
index 0000000..2ca978b
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithConstructorInvocation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Invokes a Hadoop type constructor.
+ */
+@SuppressWarnings("unused")
+public class WithConstructorInvocation {
+ /** */
+ private void foo() {
+ Object x = new Configuration();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithExtends.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithExtends.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithExtends.java
new file mode 100644
index 0000000..a0c659d
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithExtends.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.LocalFileSystem;
+
+/**
+ * Class extends a Hadoop class.
+ */
+public class WithExtends extends LocalFileSystem {
+ // noop
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithField.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithField.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithField.java
new file mode 100644
index 0000000..59c3218
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithField.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Has a Hadoop field.
+ */
+@SuppressWarnings("unused")
+public class WithField {
+ /** */
+ private Configuration conf;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithImplements.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithImplements.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithImplements.java
new file mode 100644
index 0000000..4309170
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithImplements.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Implements a Hadoop interface.
+ */
+public class WithImplements implements Configurable {
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ // noop
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithIndirectField.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithIndirectField.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithIndirectField.java
new file mode 100644
index 0000000..b4b246c
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithIndirectField.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+/**
+ * Has a unidirected dependency on Hadoop-dependent class.
+ */
+@SuppressWarnings("unused")
+public class WithIndirectField {
+ /** */
+ WithField x;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInitializer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInitializer.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInitializer.java
new file mode 100644
index 0000000..7225254
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInitializer.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+/**
+ * Has a field initialized with an expression invoking Hadoop method.
+ */
+
+@SuppressWarnings({"ConstantConditions", "unused"})
+public class WithInitializer {
+ /** */
+ private final Object x = org.apache.hadoop.fs.FileSystem.getDefaultUri(null);
+
+ /** */
+ WithInitializer() throws Exception {
+ // noop
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInnerClass.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInnerClass.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInnerClass.java
new file mode 100644
index 0000000..5c12357
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithInnerClass.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * Has a *static* inner class depending on Hadoop.
+ */
+@SuppressWarnings("unused")
+public class WithInnerClass {
+ /** */
+ private static abstract class Foo implements Configurable {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithLocalVariable.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithLocalVariable.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithLocalVariable.java
new file mode 100644
index 0000000..dde3cf7
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithLocalVariable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Has a local variable of Hadoop type.
+ */
+@SuppressWarnings({"unused", "ConstantConditions"})
+public class WithLocalVariable {
+ /** */
+ void foo() {
+ Configuration c = null;
+
+ moo(c);
+ }
+
+ /** */
+ void moo(Object x) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodAnnotation.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodAnnotation.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodAnnotation.java
new file mode 100644
index 0000000..8fe6786
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodAnnotation.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Method has a Hadoop annotation.
+ */
+@SuppressWarnings("unused")
+public class WithMethodAnnotation {
+ /** */
+ @InterfaceStability.Unstable
+ void foo() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodArgument.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodArgument.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodArgument.java
new file mode 100644
index 0000000..53a940a
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodArgument.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Contains a formal parameter of Hadoop type.
+ */
+@SuppressWarnings("unused")
+public class WithMethodArgument {
+ /** */
+ protected void paramaterMethod(Configuration c) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodCheckedException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodCheckedException.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodCheckedException.java
new file mode 100644
index 0000000..a1637c4
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodCheckedException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * Method declares a checked Hadoop Exception.
+ */
+@SuppressWarnings("unused")
+public class WithMethodCheckedException {
+ /** */
+ void foo() throws ChecksumException {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodInvocation.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodInvocation.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodInvocation.java
new file mode 100644
index 0000000..e37200c
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodInvocation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Method contains a Hadoop type method invocation.
+ */
+@SuppressWarnings("unused")
+public class WithMethodInvocation {
+ /** */
+ void foo(FileSystem fs) {
+ fs.getChildFileSystems();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodReturnType.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodReturnType.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodReturnType.java
new file mode 100644
index 0000000..5c04ed6
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodReturnType.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Contains a method return value of Hadoop type.
+ */
+@SuppressWarnings("unused")
+public class WithMethodReturnType {
+ /** */
+ FileSystem fsMethod() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodRuntimeException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodRuntimeException.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodRuntimeException.java
new file mode 100644
index 0000000..895d494
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithMethodRuntimeException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+
+/**
+ * Method declares a runtime Hadoop Exception.
+ */
+@SuppressWarnings("unused")
+public class WithMethodRuntimeException {
+ /** */
+ void foo() throws HadoopIllegalArgumentException {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithOuterClass.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithOuterClass.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithOuterClass.java
new file mode 100644
index 0000000..8c11c20
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithOuterClass.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Outer class depends on Hadoop, but Inner *static* one does not.
+ */
+@SuppressWarnings("unused")
+public class WithOuterClass {
+ /** */
+ Configuration c;
+
+ /** */
+ public static class InnerNoHadoop {
+ /** */
+ int x;
+
+ /** */
+ void foo() {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithParameterAnnotation.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithParameterAnnotation.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithParameterAnnotation.java
new file mode 100644
index 0000000..07019cc
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithParameterAnnotation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Has a paramater annotated with a Hadoop annotation.
+ */
+@SuppressWarnings("unused")
+public class WithParameterAnnotation {
+ /** */
+ void foo(@InterfaceStability.Stable Object annotatedParam) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticField.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticField.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticField.java
new file mode 100644
index 0000000..08145df
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticField.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Has a static field of Hadoop type.
+ */
+@SuppressWarnings("unused")
+public class WithStaticField {
+ /** */
+ static FileSystem fs;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticInitializer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticInitializer.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticInitializer.java
new file mode 100644
index 0000000..9b10b94
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/WithStaticInitializer.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+import java.util.List;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Uses Hadoop type in a static initializer.
+ */
+@SuppressWarnings("unused")
+public class WithStaticInitializer {
+ /** */
+ static final List x;
+
+ static {
+ x = FileSystem.getAllStatistics();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/Without.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/Without.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/Without.java
new file mode 100644
index 0000000..da1dede
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/deps/Without.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.deps;
+
+/**
+ * Class that does not anyhow depend on Hadoop.
+ */
+public class Without {
+ // No-op.
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1.java
new file mode 100644
index 0000000..0df9c6a
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+/**
+ * Example job for testing hadoop task execution.
+ */
+public class HadoopWordCount1 {
+ /**
+ * Entry point to start job.
+ * @param args command line parameters.
+ * @throws Exception if fails.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.out.println("usage: [input] [output]");
+ System.exit(-1);
+ }
+
+ JobConf job = getJob(args[0], args[1]);
+
+ JobClient.runJob(job);
+ }
+
+ /**
+ * Gets fully configured JobConf instance.
+ *
+ * @param input input file name.
+ * @param output output directory name.
+ * @return Job configuration
+ */
+ public static JobConf getJob(String input, String output) {
+ JobConf conf = new JobConf(HadoopWordCount1.class);
+ conf.setJobName("wordcount");
+
+ conf.setOutputKeyClass(Text.class);
+ conf.setOutputValueClass(IntWritable.class);
+
+ setTasksClasses(conf, true, true, true);
+
+ FileInputFormat.setInputPaths(conf, new Path(input));
+ FileOutputFormat.setOutputPath(conf, new Path(output));
+
+ return conf;
+ }
+
+ /**
+ * Sets task classes with related info if needed into configuration object.
+ *
+ * @param jobConf Configuration to change.
+ * @param setMapper Option to set mapper and input format classes.
+ * @param setCombiner Option to set combiner class.
+ * @param setReducer Option to set reducer and output format classes.
+ */
+ public static void setTasksClasses(JobConf jobConf, boolean setMapper, boolean setCombiner, boolean setReducer) {
+ if (setMapper) {
+ jobConf.setMapperClass(HadoopWordCount1Map.class);
+ jobConf.setInputFormat(TextInputFormat.class);
+ }
+
+ if (setCombiner)
+ jobConf.setCombinerClass(HadoopWordCount1Reduce.class);
+
+ if (setReducer) {
+ jobConf.setReducerClass(HadoopWordCount1Reduce.class);
+ jobConf.setOutputFormat(TextOutputFormat.class);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Map.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Map.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Map.java
new file mode 100644
index 0000000..6a98a24
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Map.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopErrorSimulator;
+
+/**
+ * Mapper phase of WordCount job.
+ */
+public class HadoopWordCount1Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** {@inheritDoc} */
+ @Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+
+ assert wasConfigured : "Mapper should be configured";
+
+ String line = val.toString();
+
+ StringTokenizer tokenizer = new StringTokenizer(line);
+
+ while (tokenizer.hasMoreTokens()) {
+ word.set(tokenizer.nextToken());
+
+ output.collect(word, one);
+ }
+
+ HadoopErrorSimulator.instance().onMap();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ super.configure(job);
+
+ wasConfigured = true;
+
+ HadoopErrorSimulator.instance().onMapConfigure();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ super.close();
+
+ HadoopErrorSimulator.instance().onMapClose();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Reduce.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Reduce.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Reduce.java
new file mode 100644
index 0000000..ab91e0c
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount1Reduce.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopErrorSimulator;
+
+/**
+ * Combiner and Reducer phase of WordCount job.
+ */
+public class HadoopWordCount1Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** {@inheritDoc} */
+ @Override public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+ assert wasConfigured : "Reducer should be configured";
+
+ int sum = 0;
+
+ while (values.hasNext())
+ sum += values.next().get();
+
+ output.collect(key, new IntWritable(sum));
+
+ HadoopErrorSimulator.instance().onReduce();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ super.configure(job);
+
+ wasConfigured = true;
+
+ HadoopErrorSimulator.instance().onReduceConfigure();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2.java
new file mode 100644
index 0000000..3ddc923
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+
+/**
+ * Example job for testing hadoop task execution.
+ */
+public class HadoopWordCount2 {
+ /**
+ * Entry point to start job.
+ *
+ * @param args Command line parameters.
+ * @throws Exception If fails.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.out.println("usage: [input] [output]");
+ System.exit(-1);
+ }
+
+ Job job = getJob(args[0], args[1]);
+
+ job.submit();
+ }
+
+ /**
+ * Gets fully configured Job instance.
+ *
+ * @param input Input file name.
+ * @param output Output directory name.
+ * @return Job instance.
+ * @throws IOException If fails.
+ */
+ public static Job getJob(String input, String output) throws IOException {
+ Job job = Job.getInstance();
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ setTasksClasses(job, true, true, true, false);
+
+ FileInputFormat.setInputPaths(job, new Path(input));
+ FileOutputFormat.setOutputPath(job, new Path(output));
+
+ job.setJarByClass(HadoopWordCount2.class);
+
+ return job;
+ }
+
+ /**
+ * Sets task classes with related info if needed into configuration object.
+ *
+ * @param job Configuration to change.
+ * @param setMapper Option to set mapper and input format classes.
+ * @param setCombiner Option to set combiner class.
+ * @param setReducer Option to set reducer and output format classes.
+ */
+ public static void setTasksClasses(Job job, boolean setMapper, boolean setCombiner, boolean setReducer,
+ boolean outputCompression) {
+ if (setMapper) {
+ job.setMapperClass(HadoopWordCount2Mapper.class);
+ job.setInputFormatClass(TextInputFormat.class);
+ }
+
+ if (setCombiner)
+ job.setCombinerClass(HadoopWordCount2Combiner.class);
+
+ if (setReducer) {
+ job.setReducerClass(HadoopWordCount2Reducer.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ }
+
+ if (outputCompression) {
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+
+ SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);
+
+ SequenceFileOutputFormat.setCompressOutput(job, true);
+
+ job.getConfiguration().set(FileOutputFormat.COMPRESS_CODEC, SnappyCodec.class.getName());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Combiner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Combiner.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Combiner.java
new file mode 100644
index 0000000..a643a92
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Combiner.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import java.io.IOException;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopErrorSimulator;
+
+/**
+ * Combiner function with pluggable error simulator.
+ */
+public class HadoopWordCount2Combiner extends HadoopWordCount2Reducer {
+ /** {@inheritDoc} */
+ @Override protected void configError() {
+ HadoopErrorSimulator.instance().onCombineConfigure();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void setupError() throws IOException, InterruptedException {
+ HadoopErrorSimulator.instance().onCombineSetup();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void reduceError() throws IOException, InterruptedException {
+ HadoopErrorSimulator.instance().onCombine();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void cleanupError() throws IOException, InterruptedException {
+ HadoopErrorSimulator.instance().onCombineCleanup();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Mapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Mapper.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Mapper.java
new file mode 100644
index 0000000..336db84
--- /dev/null
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/examples/HadoopWordCount2Mapper.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.examples;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopErrorSimulator;
+
+/**
+ * Mapper phase of WordCount job.
+ */
+public class HadoopWordCount2Mapper extends Mapper<Object, Text, Text, IntWritable> implements Configurable {
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** Flag is to check that mapper was set up before run. */
+ private boolean wasSetUp;
+
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ assert wasConfigured : "Mapper should be configured";
+ assert wasSetUp : "Mapper should be set up";
+
+ StringTokenizer wordList = new StringTokenizer(val.toString());
+
+ while (wordList.hasMoreTokens()) {
+ word.set(wordList.nextToken());
+
+ ctx.write(word, one);
+ }
+
+ HadoopErrorSimulator.instance().onMap();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ super.setup(ctx);
+
+ wasSetUp = true;
+
+ HadoopErrorSimulator.instance().onMapSetup();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void cleanup(Context ctx) throws IOException, InterruptedException {
+ super.cleanup(ctx);
+
+ HadoopErrorSimulator.instance().onMapCleanup();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ wasConfigured = true;
+
+ HadoopErrorSimulator.instance().onMapConfigure();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+}
\ No newline at end of file